2024-11-26 14:14:43,747 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 14:14:43,764 main DEBUG Took 0.013948 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 14:14:43,764 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 14:14:43,765 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 14:14:43,766 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 14:14:43,768 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,778 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 14:14:43,794 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,797 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,798 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,800 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,802 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,803 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,806 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,807 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,808 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,809 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:14:43,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,811 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 14:14:43,813 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:14:43,815 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 14:14:43,817 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 14:14:43,818 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 14:14:43,819 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 14:14:43,820 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 14:14:43,830 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 14:14:43,833 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 14:14:43,835 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 14:14:43,835 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 14:14:43,836 main DEBUG createAppenders(={Console}) 2024-11-26 14:14:43,837 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 14:14:43,837 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 14:14:43,838 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 14:14:43,838 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 14:14:43,839 main DEBUG OutputStream closed 2024-11-26 14:14:43,839 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 14:14:43,839 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 14:14:43,840 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 14:14:43,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 14:14:43,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 14:14:43,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 14:14:43,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 14:14:43,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 14:14:43,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 14:14:43,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 14:14:43,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 14:14:43,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 14:14:43,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 14:14:43,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 14:14:43,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 14:14:43,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 14:14:43,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 14:14:43,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 14:14:43,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 14:14:43,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 14:14:43,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 14:14:43,950 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 14:14:43,950 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 14:14:43,951 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 14:14:43,952 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T14:14:44,299 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca 2024-11-26 14:14:44,301 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 14:14:44,302 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T14:14:44,311 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-26T14:14:44,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T14:14:44,354 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b, deleteOnExit=true 2024-11-26T14:14:44,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T14:14:44,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/test.cache.data in system properties and HBase conf 2024-11-26T14:14:44,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T14:14:44,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.log.dir in system properties and HBase conf 2024-11-26T14:14:44,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T14:14:44,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T14:14:44,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T14:14:44,455 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T14:14:44,586 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T14:14:44,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T14:14:44,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T14:14:44,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T14:14:44,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T14:14:44,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T14:14:44,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T14:14:44,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T14:14:44,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T14:14:44,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T14:14:44,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/nfs.dump.dir in system properties and HBase conf 2024-11-26T14:14:44,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/java.io.tmpdir in system properties and HBase conf 2024-11-26T14:14:44,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T14:14:44,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T14:14:44,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T14:14:45,774 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T14:14:45,861 INFO [Time-limited test {}] log.Log(170): Logging initialized @2913ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T14:14:45,935 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:14:46,038 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:14:46,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:14:46,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:14:46,073 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T14:14:46,089 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:14:46,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:14:46,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:14:46,280 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/java.io.tmpdir/jetty-localhost-41747-hadoop-hdfs-3_4_1-tests_jar-_-any-2862867557802548975/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T14:14:46,295 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41747} 2024-11-26T14:14:46,295 INFO [Time-limited test {}] server.Server(415): Started @3348ms 2024-11-26T14:14:46,826 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:14:46,834 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:14:46,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:14:46,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:14:46,835 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T14:14:46,836 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:14:46,837 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:14:46,953 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/java.io.tmpdir/jetty-localhost-46537-hadoop-hdfs-3_4_1-tests_jar-_-any-5340448647407123525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:14:46,954 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:46537} 2024-11-26T14:14:46,955 INFO [Time-limited test {}] server.Server(415): Started @4008ms 2024-11-26T14:14:47,015 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T14:14:48,228 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data1/current/BP-17547768-172.17.0.3-1732630485263/current, will proceed with Du for space computation calculation, 2024-11-26T14:14:48,229 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data2/current/BP-17547768-172.17.0.3-1732630485263/current, will proceed with Du for space computation calculation, 2024-11-26T14:14:48,295 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T14:14:48,376 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8049bc74f4dd3e52 with lease ID 0xde90c6fe2b16fe5f: Processing first storage report for DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34 from datanode DatanodeRegistration(127.0.0.1:46501, datanodeUuid=16b1d143-87c6-4d34-abc0-c7270ac6fb80, infoPort=33563, infoSecurePort=0, ipcPort=39359, storageInfo=lv=-57;cid=testClusterID;nsid=92414906;c=1732630485263) 2024-11-26T14:14:48,379 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8049bc74f4dd3e52 with lease ID 0xde90c6fe2b16fe5f: from storage DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34 node DatanodeRegistration(127.0.0.1:46501, datanodeUuid=16b1d143-87c6-4d34-abc0-c7270ac6fb80, infoPort=33563, infoSecurePort=0, ipcPort=39359, storageInfo=lv=-57;cid=testClusterID;nsid=92414906;c=1732630485263), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T14:14:48,380 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8049bc74f4dd3e52 with lease ID 0xde90c6fe2b16fe5f: Processing first storage report for DS-d92d192c-f9ba-4cb1-b41c-c1befb6bd5c5 from datanode DatanodeRegistration(127.0.0.1:46501, datanodeUuid=16b1d143-87c6-4d34-abc0-c7270ac6fb80, infoPort=33563, infoSecurePort=0, ipcPort=39359, storageInfo=lv=-57;cid=testClusterID;nsid=92414906;c=1732630485263) 2024-11-26T14:14:48,380 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8049bc74f4dd3e52 with lease ID 0xde90c6fe2b16fe5f: from storage DS-d92d192c-f9ba-4cb1-b41c-c1befb6bd5c5 node DatanodeRegistration(127.0.0.1:46501, datanodeUuid=16b1d143-87c6-4d34-abc0-c7270ac6fb80, infoPort=33563, infoSecurePort=0, ipcPort=39359, storageInfo=lv=-57;cid=testClusterID;nsid=92414906;c=1732630485263), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T14:14:48,384 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca 2024-11-26T14:14:48,524 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/zookeeper_0, clientPort=58426, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T14:14:48,546 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58426 2024-11-26T14:14:48,579 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:48,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:48,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741825_1001 (size=7) 2024-11-26T14:14:49,332 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc with version=8 2024-11-26T14:14:49,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/hbase-staging 2024-11-26T14:14:49,471 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T14:14:49,842 INFO [Time-limited test {}] client.ConnectionUtils(128): master/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:14:49,856 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:49,856 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:49,863 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:14:49,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:49,864 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:14:50,145 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T14:14:50,233 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T14:14:50,246 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T14:14:50,252 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:14:50,286 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4751 (auto-detected) 2024-11-26T14:14:50,288 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T14:14:50,324 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37705 2024-11-26T14:14:50,379 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37705 connecting to ZooKeeper ensemble=127.0.0.1:58426 2024-11-26T14:14:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377050x0, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:14:50,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37705-0x101777f21520000 connected 2024-11-26T14:14:50,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:50,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:50,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:14:50,838 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc, hbase.cluster.distributed=false 2024-11-26T14:14:50,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:14:50,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37705 2024-11-26T14:14:50,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37705 2024-11-26T14:14:50,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37705 2024-11-26T14:14:50,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37705 2024-11-26T14:14:50,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37705 2024-11-26T14:14:51,048 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:14:51,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:51,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:51,053 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:14:51,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:14:51,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:14:51,058 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T14:14:51,061 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:14:51,069 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40025 2024-11-26T14:14:51,073 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40025 connecting to ZooKeeper ensemble=127.0.0.1:58426 2024-11-26T14:14:51,080 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:51,089 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:51,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400250x0, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:14:51,192 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40025-0x101777f21520001 connected 2024-11-26T14:14:51,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:14:51,209 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T14:14:51,242 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T14:14:51,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T14:14:51,280 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:14:51,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-26T14:14:51,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40025 2024-11-26T14:14:51,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40025 2024-11-26T14:14:51,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-26T14:14:51,310 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-26T14:14:51,347 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;19945b63c2b4:37705 2024-11-26T14:14:51,356 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/19945b63c2b4,37705,1732630489590 2024-11-26T14:14:51,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:14:51,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:14:51,441 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/19945b63c2b4,37705,1732630489590 2024-11-26T14:14:51,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T14:14:51,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:51,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:51,502 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T14:14:51,503 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/19945b63c2b4,37705,1732630489590 from backup master directory 2024-11-26T14:14:51,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/19945b63c2b4,37705,1732630489590 2024-11-26T14:14:51,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:14:51,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:14:51,525 WARN [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:14:51,526 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=19945b63c2b4,37705,1732630489590 2024-11-26T14:14:51,528 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T14:14:51,536 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T14:14:51,617 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/hbase.id] with ID: 4854e9f7-b361-4b2a-9f10-47c2bb43584f 2024-11-26T14:14:51,617 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/.tmp/hbase.id 2024-11-26T14:14:51,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741826_1002 (size=42) 2024-11-26T14:14:52,038 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/.tmp/hbase.id]:[hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/hbase.id] 2024-11-26T14:14:52,099 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:14:52,106 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T14:14:52,133 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-26T14:14:52,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:52,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:52,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741827_1003 (size=196) 2024-11-26T14:14:52,261 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:14:52,265 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T14:14:52,290 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:52,296 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:14:52,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741828_1004 (size=1189) 2024-11-26T14:14:52,788 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store 2024-11-26T14:14:52,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741829_1005 (size=34) 2024-11-26T14:14:52,826 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T14:14:52,830 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:52,831 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T14:14:52,832 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:14:52,832 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:14:52,834 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T14:14:52,834 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:14:52,835 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:14:52,836 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732630492831Disabling compacts and flushes for region at 1732630492831Disabling writes for close at 1732630492834 (+3 ms)Writing region close event to WAL at 1732630492835 (+1 ms)Closed at 1732630492835 2024-11-26T14:14:52,841 WARN [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/.initializing 2024-11-26T14:14:52,841 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/WALs/19945b63c2b4,37705,1732630489590 2024-11-26T14:14:52,852 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:14:52,875 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C37705%2C1732630489590, suffix=, logDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/WALs/19945b63c2b4,37705,1732630489590, archiveDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/oldWALs, maxLogs=10 2024-11-26T14:14:52,909 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/WALs/19945b63c2b4,37705,1732630489590/19945b63c2b4%2C37705%2C1732630489590.1732630492881, exclude list is [], retry=0 2024-11-26T14:14:52,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK] 2024-11-26T14:14:52,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T14:14:53,004 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/WALs/19945b63c2b4,37705,1732630489590/19945b63c2b4%2C37705%2C1732630489590.1732630492881 2024-11-26T14:14:53,006 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33563:33563)] 2024-11-26T14:14:53,009 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:14:53,009 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:53,017 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,020 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T14:14:53,126 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:53,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:53,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T14:14:53,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:53,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:53,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T14:14:53,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:53,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:53,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T14:14:53,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:53,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:53,158 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,166 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,169 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,179 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,180 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,189 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T14:14:53,199 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:14:53,213 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:14:53,215 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74543400, jitterRate=0.11078321933746338}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T14:14:53,222 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732630493037Initializing all the Stores at 1732630493045 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630493045Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630493048 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630493049 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630493049Cleaning up temporary data from old regions at 1732630493181 (+132 ms)Region opened successfully at 1732630493222 (+41 ms) 2024-11-26T14:14:53,229 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T14:14:53,298 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47899d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:14:53,344 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T14:14:53,363 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T14:14:53,363 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T14:14:53,367 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T14:14:53,369 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T14:14:53,377 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-26T14:14:53,378 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T14:14:53,411 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T14:14:53,428 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T14:14:53,520 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T14:14:53,524 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T14:14:53,526 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T14:14:53,539 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T14:14:53,542 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T14:14:53,547 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T14:14:53,560 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T14:14:53,562 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T14:14:53,573 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T14:14:53,598 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T14:14:53,605 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T14:14:53,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:14:53,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:14:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,619 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=19945b63c2b4,37705,1732630489590, sessionid=0x101777f21520000, setting cluster-up flag (Was=false) 2024-11-26T14:14:53,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,680 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T14:14:53,685 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=19945b63c2b4,37705,1732630489590 2024-11-26T14:14:53,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:53,741 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T14:14:53,743 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=19945b63c2b4,37705,1732630489590 2024-11-26T14:14:53,751 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T14:14:53,828 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(746): ClusterId : 4854e9f7-b361-4b2a-9f10-47c2bb43584f 2024-11-26T14:14:53,832 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T14:14:53,851 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T14:14:53,851 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T14:14:53,869 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T14:14:53,885 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T14:14:53,886 DEBUG [RS:0;19945b63c2b4:40025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755102b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:14:53,886 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T14:14:53,897 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T14:14:53,909 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;19945b63c2b4:40025 2024-11-26T14:14:53,907 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 19945b63c2b4,37705,1732630489590 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T14:14:53,919 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T14:14:53,919 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T14:14:53,919 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T14:14:53,923 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:14:53,923 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:14:53,924 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:14:53,926 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,37705,1732630489590 with port=40025, startcode=1732630490995 2024-11-26T14:14:53,932 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:14:53,933 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/19945b63c2b4:0, corePoolSize=10, maxPoolSize=10 2024-11-26T14:14:53,933 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:53,933 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:14:53,933 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:53,944 DEBUG [RS:0;19945b63c2b4:40025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T14:14:53,977 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:14:53,978 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T14:14:53,978 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732630523978 2024-11-26T14:14:53,983 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T14:14:53,985 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T14:14:53,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:53,995 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T14:14:53,998 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T14:14:53,998 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T14:14:54,021 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T14:14:54,022 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T14:14:54,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T14:14:54,025 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,046 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37705 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:54,057 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T14:14:54,058 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T14:14:54,058 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T14:14:54,062 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T14:14:54,062 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T14:14:54,068 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732630494064,5,FailOnTimeoutGroup] 2024-11-26T14:14:54,076 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732630494069,5,FailOnTimeoutGroup] 2024-11-26T14:14:54,077 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,077 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T14:14:54,079 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,079 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741831_1007 (size=1321) 2024-11-26T14:14:54,122 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T14:14:54,123 WARN [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T14:14:54,224 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,37705,1732630489590 with port=40025, startcode=1732630490995 2024-11-26T14:14:54,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37705 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,230 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37705 {}] master.ServerManager(517): Registering regionserver=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,241 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:14:54,242 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46747 2024-11-26T14:14:54,242 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T14:14:54,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:14:54,290 DEBUG [RS:0;19945b63c2b4:40025 {}] zookeeper.ZKUtil(111): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,290 WARN [RS:0;19945b63c2b4:40025 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:14:54,290 INFO [RS:0;19945b63c2b4:40025 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:14:54,291 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,292 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [19945b63c2b4,40025,1732630490995] 2024-11-26T14:14:54,339 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T14:14:54,362 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T14:14:54,369 INFO [RS:0;19945b63c2b4:40025 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:14:54,369 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,373 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T14:14:54,380 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T14:14:54,383 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,383 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,383 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,383 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,383 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,384 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,384 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:14:54,384 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,384 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:14:54,385 DEBUG [RS:0;19945b63c2b4:40025 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:14:54,396 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,397 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,397 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,397 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,397 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,397 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,40025,1732630490995-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:14:54,433 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T14:14:54,436 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,40025,1732630490995-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,437 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,437 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.Replication(171): 19945b63c2b4,40025,1732630490995 started 2024-11-26T14:14:54,475 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:54,476 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1482): Serving as 19945b63c2b4,40025,1732630490995, RpcServer on 19945b63c2b4/172.17.0.3:40025, sessionid=0x101777f21520001 2024-11-26T14:14:54,477 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T14:14:54,477 DEBUG [RS:0;19945b63c2b4:40025 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,478 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,40025,1732630490995' 2024-11-26T14:14:54,478 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T14:14:54,484 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T14:14:54,500 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T14:14:54,501 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T14:14:54,501 DEBUG [RS:0;19945b63c2b4:40025 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 19945b63c2b4,40025,1732630490995 2024-11-26T14:14:54,501 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,40025,1732630490995' 2024-11-26T14:14:54,501 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T14:14:54,512 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T14:14:54,521 DEBUG [RS:0;19945b63c2b4:40025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T14:14:54,521 INFO [RS:0;19945b63c2b4:40025 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T14:14:54,521 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T14:14:54,521 INFO [RS:0;19945b63c2b4:40025 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T14:14:54,522 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:14:54,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741832_1008 (size=32) 2024-11-26T14:14:54,628 INFO [RS:0;19945b63c2b4:40025 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:14:54,633 INFO [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C40025%2C1732630490995, suffix=, logDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995, archiveDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/oldWALs, maxLogs=32 2024-11-26T14:14:54,660 DEBUG [RS:0;19945b63c2b4:40025 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995/19945b63c2b4%2C40025%2C1732630490995.1732630494636, exclude list is [], retry=0 2024-11-26T14:14:54,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK] 2024-11-26T14:14:54,682 INFO [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995/19945b63c2b4%2C40025%2C1732630490995.1732630494636 2024-11-26T14:14:54,685 DEBUG [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33563:33563)] 2024-11-26T14:14:54,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:54,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T14:14:54,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T14:14:54,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:54,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:54,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T14:14:54,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T14:14:54,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:54,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:54,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T14:14:54,988 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T14:14:54,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:54,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:54,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T14:14:54,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T14:14:54,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:54,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:54,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T14:14:54,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740 2024-11-26T14:14:55,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740 2024-11-26T14:14:55,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T14:14:55,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T14:14:55,006 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:14:55,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T14:14:55,017 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:14:55,021 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60910916, jitterRate=-0.0923566222190857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:14:55,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732630494969Initializing all the Stores at 1732630494972 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630494972Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630494972Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630494972Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630494972Cleaning up temporary data from old regions at 1732630495005 (+33 ms)Region opened successfully at 1732630495024 (+19 ms) 2024-11-26T14:14:55,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T14:14:55,025 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T14:14:55,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T14:14:55,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T14:14:55,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T14:14:55,027 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T14:14:55,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732630495024Disabling compacts and flushes for region at 1732630495024Disabling writes for close at 1732630495025 (+1 ms)Writing region close event to WAL at 1732630495027 (+2 ms)Closed at 1732630495027 2024-11-26T14:14:55,033 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:14:55,033 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T14:14:55,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T14:14:55,056 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T14:14:55,060 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T14:14:55,214 DEBUG [19945b63c2b4:37705 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T14:14:55,231 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:55,239 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 19945b63c2b4,40025,1732630490995, state=OPENING 2024-11-26T14:14:55,257 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T14:14:55,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:55,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:14:55,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:14:55,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:14:55,270 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T14:14:55,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:14:55,454 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:14:55,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39567, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:14:55,478 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T14:14:55,478 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:14:55,479 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T14:14:55,489 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C40025%2C1732630490995.meta, suffix=.meta, logDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995, archiveDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/oldWALs, maxLogs=32 2024-11-26T14:14:55,512 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995/19945b63c2b4%2C40025%2C1732630490995.meta.1732630495491.meta, exclude list is [], retry=0 2024-11-26T14:14:55,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK] 2024-11-26T14:14:55,533 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/WALs/19945b63c2b4,40025,1732630490995/19945b63c2b4%2C40025%2C1732630490995.meta.1732630495491.meta 2024-11-26T14:14:55,536 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33563:33563)] 2024-11-26T14:14:55,537 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:14:55,539 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T14:14:55,543 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T14:14:55,556 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T14:14:55,562 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T14:14:55,563 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:55,564 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T14:14:55,564 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T14:14:55,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T14:14:55,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T14:14:55,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:55,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:55,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T14:14:55,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T14:14:55,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:55,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:55,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T14:14:55,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T14:14:55,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:55,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:55,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T14:14:55,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T14:14:55,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:55,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:14:55,605 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T14:14:55,608 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740 2024-11-26T14:14:55,613 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740 2024-11-26T14:14:55,618 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T14:14:55,619 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T14:14:55,620 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:14:55,629 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T14:14:55,636 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62060016, jitterRate=-0.07523369789123535}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:14:55,637 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T14:14:55,639 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732630495565Writing region info on filesystem at 1732630495565Initializing all the Stores at 1732630495572 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630495572Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630495576 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630495576Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732630495576Cleaning up temporary data from old regions at 1732630495619 (+43 ms)Running coprocessor post-open hooks at 1732630495637 (+18 ms)Region opened successfully at 1732630495639 (+2 ms) 2024-11-26T14:14:55,649 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732630495443 2024-11-26T14:14:55,666 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T14:14:55,666 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T14:14:55,669 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:55,673 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 19945b63c2b4,40025,1732630490995, state=OPEN 2024-11-26T14:14:55,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:14:55,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:14:55,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:14:55,720 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:55,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:14:55,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T14:14:55,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=19945b63c2b4,40025,1732630490995 in 448 msec 2024-11-26T14:14:55,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T14:14:55,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 692 msec 2024-11-26T14:14:55,767 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:14:55,767 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T14:14:55,812 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:55,814 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:55,844 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:55,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36275, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:55,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.0790 sec 2024-11-26T14:14:55,880 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732630495880, completionTime=-1 2024-11-26T14:14:55,884 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T14:14:55,884 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T14:14:55,919 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T14:14:55,919 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732630555919 2024-11-26T14:14:55,919 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732630615919 2024-11-26T14:14:55,920 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-11-26T14:14:55,922 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,923 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,923 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,925 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-19945b63c2b4:37705, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,926 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,926 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:55,935 DEBUG [master/19945b63c2b4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T14:14:55,962 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.436sec 2024-11-26T14:14:55,964 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T14:14:55,965 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T14:14:55,966 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T14:14:55,967 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T14:14:55,967 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T14:14:55,968 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:14:55,968 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T14:14:55,982 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T14:14:55,983 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T14:14:55,984 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37705,1732630489590-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:14:56,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:56,053 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T14:14:56,053 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T14:14:56,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:56,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:56,084 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72188503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:56,105 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:56,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:56,117 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:56,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:56,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=19945b63c2b4,37705,1732630489590 2024-11-26T14:14:56,193 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=795, ProcessCount=11, AvailableMemoryMB=4498 2024-11-26T14:14:56,209 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:14:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:14:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a30b8b7 2024-11-26T14:14:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:14:56,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:14:56,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:14:56,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:14:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:14:56,281 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:14:56,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T14:14:56,285 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:56,290 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:14:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:14:56,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741835_1011 (size=1061) 2024-11-26T14:14:56,374 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:14:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:14:56,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741836_1012 (size=53) 2024-11-26T14:14:56,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:56,448 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c77bbfdf456d364621817f40b0d9aed2, disabling compactions & flushes 2024-11-26T14:14:56,448 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. after waiting 0 ms 2024-11-26T14:14:56,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,453 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,453 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c77bbfdf456d364621817f40b0d9aed2: Waiting for close lock at 1732630496448Disabling compacts and flushes for region at 1732630496448Disabling writes for close at 1732630496453 (+5 ms)Writing region close event to WAL at 1732630496453Closed at 1732630496453 2024-11-26T14:14:56,458 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:14:56,466 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630496459"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630496459"}]},"ts":"1732630496459"} 2024-11-26T14:14:56,478 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:14:56,483 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:14:56,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630496483"}]},"ts":"1732630496483"} 2024-11-26T14:14:56,542 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:14:56,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, ASSIGN}] 2024-11-26T14:14:56,556 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, ASSIGN 2024-11-26T14:14:56,566 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:14:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:14:56,719 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c77bbfdf456d364621817f40b0d9aed2, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:56,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, ASSIGN because future has completed 2024-11-26T14:14:56,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:14:56,899 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,900 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:14:56,901 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,901 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:14:56,901 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,901 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,909 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,915 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:14:56,915 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c77bbfdf456d364621817f40b0d9aed2 columnFamilyName A 2024-11-26T14:14:56,916 DEBUG [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:56,919 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(327): Store=c77bbfdf456d364621817f40b0d9aed2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:56,919 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,925 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:14:56,926 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c77bbfdf456d364621817f40b0d9aed2 columnFamilyName B 2024-11-26T14:14:56,926 DEBUG [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:56,930 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(327): Store=c77bbfdf456d364621817f40b0d9aed2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:56,930 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:14:56,946 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:14:56,947 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c77bbfdf456d364621817f40b0d9aed2 columnFamilyName C 2024-11-26T14:14:56,947 DEBUG [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:14:56,949 INFO [StoreOpener-c77bbfdf456d364621817f40b0d9aed2-1 {}] regionserver.HStore(327): Store=c77bbfdf456d364621817f40b0d9aed2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:14:56,950 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:56,951 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,953 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,955 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,961 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,961 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,962 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:14:56,967 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,983 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:14:56,985 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c77bbfdf456d364621817f40b0d9aed2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65901856, jitterRate=-0.017985820770263672}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:14:56,985 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:56,987 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c77bbfdf456d364621817f40b0d9aed2: Running coprocessor pre-open hook at 1732630496901Writing region info on filesystem at 1732630496901Initializing all the Stores at 1732630496908 (+7 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630496908Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630496908Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630496908Cleaning up temporary data from old regions at 1732630496961 (+53 ms)Running coprocessor post-open hooks at 1732630496985 (+24 ms)Region opened successfully at 1732630496987 (+2 ms) 2024-11-26T14:14:56,996 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., pid=6, masterSystemTime=1732630496887 2024-11-26T14:14:57,010 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:57,010 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:57,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c77bbfdf456d364621817f40b0d9aed2, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:57,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:14:57,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T14:14:57,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 in 310 msec 2024-11-26T14:14:57,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T14:14:57,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, ASSIGN in 504 msec 2024-11-26T14:14:57,093 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:14:57,093 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630497093"}]},"ts":"1732630497093"} 2024-11-26T14:14:57,101 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:14:57,112 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:14:57,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 855 msec 2024-11-26T14:14:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:14:57,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:14:57,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c32821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dee243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5efef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,528 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:14:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:14:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f48350e 2024-11-26T14:14:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:14:57,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:14:57,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,628 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,653 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c2183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a1730c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb793a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60658e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af29a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a18c602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b9987f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,801 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44f32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486c0b29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:14:57,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:14:57,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:14:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:14:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:14:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c17951c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:14:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:14:57,823 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:14:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b49bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:14:57,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:14:57,856 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,857 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,857 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,859 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,860 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,860 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,861 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,865 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,866 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:14:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,870 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,870 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:14:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:14:57,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T14:14:57,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:14:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:14:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:14:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,904 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:14:57,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:14:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:14:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:14:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:58,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:14:58,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:14:58,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:58,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:14:58,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:58,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:14:58,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:58,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:58,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:58,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:14:58,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:58,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630558353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630558362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630558377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630558385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630558386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/5900687dd65f4432be564bc96dc94688 is 50, key is test_row_0/A:col10/1732630498075/Put/seqid=0 2024-11-26T14:14:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741837_1013 (size=16681) 2024-11-26T14:14:58,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/5900687dd65f4432be564bc96dc94688 2024-11-26T14:14:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:14:58,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:58,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630558526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630558527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630558528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630558527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630558529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/d2d12f8c2c0f44198c697e7d952be226 is 50, key is test_row_0/B:col10/1732630498075/Put/seqid=0 2024-11-26T14:14:58,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741838_1014 (size=12001) 2024-11-26T14:14:58,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/d2d12f8c2c0f44198c697e7d952be226 2024-11-26T14:14:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630558755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630558760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630558769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630558773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/ffb3d3c604ed416a8a8b7d9118b98602 is 50, key is test_row_0/C:col10/1732630498075/Put/seqid=0 2024-11-26T14:14:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630558770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741839_1015 (size=12001) 2024-11-26T14:14:58,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:58,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:59,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:14:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630559074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630559084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630559083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630559097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630559103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/ffb3d3c604ed416a8a8b7d9118b98602 2024-11-26T14:14:59,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/5900687dd65f4432be564bc96dc94688 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688 2024-11-26T14:14:59,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:59,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:14:59,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688, entries=250, sequenceid=13, filesize=16.3 K 2024-11-26T14:14:59,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/d2d12f8c2c0f44198c697e7d952be226 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226 2024-11-26T14:14:59,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T14:14:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/ffb3d3c604ed416a8a8b7d9118b98602 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602 2024-11-26T14:14:59,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T14:14:59,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c77bbfdf456d364621817f40b0d9aed2 in 1381ms, sequenceid=13, compaction requested=false 2024-11-26T14:14:59,468 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T14:14:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:14:59,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T14:14:59,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,530 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T14:14:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:14:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:14:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:14:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0e24536015a14f0f8c4b98d639c9248e is 50, key is test_row_0/A:col10/1732630498372/Put/seqid=0 2024-11-26T14:14:59,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741840_1016 (size=12001) 2024-11-26T14:14:59,613 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0e24536015a14f0f8c4b98d639c9248e 2024-11-26T14:14:59,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:59,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:14:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/e21f4c0114b542afa5668b39fbc95b02 is 50, key is test_row_0/B:col10/1732630498372/Put/seqid=0 2024-11-26T14:14:59,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741841_1017 (size=12001) 2024-11-26T14:14:59,705 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/e21f4c0114b542afa5668b39fbc95b02 2024-11-26T14:14:59,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/56249e48d62f4adaaafa5b0c71cca40e is 50, key is test_row_0/C:col10/1732630498372/Put/seqid=0 2024-11-26T14:14:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630559674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630559678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741842_1018 (size=12001) 2024-11-26T14:14:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630559705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,783 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/56249e48d62f4adaaafa5b0c71cca40e 2024-11-26T14:14:59,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0e24536015a14f0f8c4b98d639c9248e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e 2024-11-26T14:14:59,818 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T14:14:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/e21f4c0114b542afa5668b39fbc95b02 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02 2024-11-26T14:14:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630559737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630559745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T14:14:59,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/56249e48d62f4adaaafa5b0c71cca40e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e 2024-11-26T14:14:59,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630559857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:14:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630559857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:14:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:14:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:14:59,895 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T14:14:59,902 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c77bbfdf456d364621817f40b0d9aed2 in 372ms, sequenceid=33, compaction requested=false 2024-11-26T14:14:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:14:59,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:14:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T14:14:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T14:14:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:14:59,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:14:59,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:14:59,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:14:59,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:14:59,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:14:59,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T14:14:59,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-26T14:14:59,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/b3bb50645e56445e881d840d770a5c8f is 50, key is test_row_0/A:col10/1732630499725/Put/seqid=0 2024-11-26T14:14:59,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0770 sec 2024-11-26T14:15:00,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741843_1019 (size=14341) 2024-11-26T14:15:00,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/b3bb50645e56445e881d840d770a5c8f 2024-11-26T14:15:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T14:15:00,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:00,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T14:15:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:00,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:00,064 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:00,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:00,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/cf092f8c946e4758bb347a29a7015400 is 50, key is test_row_0/B:col10/1732630499725/Put/seqid=0 2024-11-26T14:15:00,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741844_1020 (size=12001) 2024-11-26T14:15:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:00,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:00,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T14:15:00,227 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T14:15:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T14:15:00,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T14:15:00,236 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T14:15:00,236 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T14:15:00,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T14:15:00,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T14:15:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630560211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630560226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630560239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630560252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630560276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,332 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T14:15:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:00,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630560384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630560393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630560394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630560404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630560404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/cf092f8c946e4758bb347a29a7015400 2024-11-26T14:15:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630560614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1f5441fbb1074ae3a30e9daacf3230f7 is 50, key is test_row_0/C:col10/1732630499725/Put/seqid=0 2024-11-26T14:15:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630560614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630560619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630560632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630560636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741845_1021 (size=12001) 2024-11-26T14:15:00,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:00,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630560934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630560934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630560943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630560954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630560964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:01,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1f5441fbb1074ae3a30e9daacf3230f7 2024-11-26T14:15:01,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/b3bb50645e56445e881d840d770a5c8f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f 2024-11-26T14:15:01,112 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:15:01,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f, entries=200, sequenceid=46, filesize=14.0 K 2024-11-26T14:15:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/cf092f8c946e4758bb347a29a7015400 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400 2024-11-26T14:15:01,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T14:15:01,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1f5441fbb1074ae3a30e9daacf3230f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7 2024-11-26T14:15:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:01,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:01,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:01,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T14:15:01,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c77bbfdf456d364621817f40b0d9aed2 in 1316ms, sequenceid=46, compaction requested=true 2024-11-26T14:15:01,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:01,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:01,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:01,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:01,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:01,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:01,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:01,246 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:01,248 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:01,259 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:01,261 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:01,262 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,262 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=42.0 K 2024-11-26T14:15:01,268 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:01,269 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:01,269 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,269 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.2 K 2024-11-26T14:15:01,277 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5900687dd65f4432be564bc96dc94688, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732630497999 2024-11-26T14:15:01,277 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d2d12f8c2c0f44198c697e7d952be226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732630497999 2024-11-26T14:15:01,288 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e21f4c0114b542afa5668b39fbc95b02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732630498356 2024-11-26T14:15:01,289 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e24536015a14f0f8c4b98d639c9248e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732630498356 2024-11-26T14:15:01,292 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bb50645e56445e881d840d770a5c8f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499651 2024-11-26T14:15:01,294 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting cf092f8c946e4758bb347a29a7015400, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499704 2024-11-26T14:15:01,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T14:15:01,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,372 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:01,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0039a9fff6f14a0dae51be20dcdf6e6e is 50, key is test_row_0/A:col10/1732630500238/Put/seqid=0 2024-11-26T14:15:01,450 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#9 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:01,457 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/6872ae8ff4124a04ad32e19cb12223ef is 50, key is test_row_0/A:col10/1732630499725/Put/seqid=0 2024-11-26T14:15:01,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741846_1022 (size=12001) 2024-11-26T14:15:01,493 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#10 average throughput is 0.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:01,498 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0039a9fff6f14a0dae51be20dcdf6e6e 2024-11-26T14:15:01,499 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3f4f3666e20e45ce98c503b6418bc4dc is 50, key is test_row_0/B:col10/1732630499725/Put/seqid=0 2024-11-26T14:15:01,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:01,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741847_1023 (size=12104) 2024-11-26T14:15:01,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741848_1024 (size=12104) 2024-11-26T14:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630561544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630561545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630561554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9ebb16d3338d4865b197e17b06c1ad96 is 50, key is test_row_0/B:col10/1732630500238/Put/seqid=0 2024-11-26T14:15:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630561561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630561566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,601 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3f4f3666e20e45ce98c503b6418bc4dc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3f4f3666e20e45ce98c503b6418bc4dc 2024-11-26T14:15:01,601 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/6872ae8ff4124a04ad32e19cb12223ef as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6872ae8ff4124a04ad32e19cb12223ef 2024-11-26T14:15:01,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741849_1025 (size=12001) 2024-11-26T14:15:01,646 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 6872ae8ff4124a04ad32e19cb12223ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:01,647 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:01,649 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630501225; duration=0sec 2024-11-26T14:15:01,650 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:01,650 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:01,650 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:01,651 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into 3f4f3666e20e45ce98c503b6418bc4dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:01,651 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:01,651 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630501245; duration=0sec 2024-11-26T14:15:01,651 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:01,651 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:01,660 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:01,660 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:01,660 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:01,661 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.2 K 2024-11-26T14:15:01,672 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb3d3c604ed416a8a8b7d9118b98602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732630497999 2024-11-26T14:15:01,674 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56249e48d62f4adaaafa5b0c71cca40e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732630498356 2024-11-26T14:15:01,675 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5441fbb1074ae3a30e9daacf3230f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499704 2024-11-26T14:15:01,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630561673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630561674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630561676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630561694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630561695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,716 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:01,717 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/798f1a5525794112858730da7b149b03 is 50, key is test_row_0/C:col10/1732630499725/Put/seqid=0 2024-11-26T14:15:01,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741850_1026 (size=12104) 2024-11-26T14:15:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630561904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630561905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630561904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630561914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630561914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,056 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9ebb16d3338d4865b197e17b06c1ad96 2024-11-26T14:15:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/9429e953358d4d1699bfeca4cf7f6b35 is 50, key is test_row_0/C:col10/1732630500238/Put/seqid=0 2024-11-26T14:15:02,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741851_1027 (size=12001) 2024-11-26T14:15:02,195 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/798f1a5525794112858730da7b149b03 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/798f1a5525794112858730da7b149b03 2024-11-26T14:15:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:02,217 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 798f1a5525794112858730da7b149b03(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:02,217 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:02,217 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630501246; duration=0sec 2024-11-26T14:15:02,218 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:02,218 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630562224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630562226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630562232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630562232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630562232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,584 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/9429e953358d4d1699bfeca4cf7f6b35 2024-11-26T14:15:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0039a9fff6f14a0dae51be20dcdf6e6e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e 2024-11-26T14:15:02,655 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e, entries=150, sequenceid=65, filesize=11.7 K 2024-11-26T14:15:02,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9ebb16d3338d4865b197e17b06c1ad96 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96 2024-11-26T14:15:02,706 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96, entries=150, sequenceid=65, filesize=11.7 K 2024-11-26T14:15:02,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/9429e953358d4d1699bfeca4cf7f6b35 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35 2024-11-26T14:15:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630562743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630562757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630562757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,765 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35, entries=150, sequenceid=65, filesize=11.7 K 2024-11-26T14:15:02,776 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c77bbfdf456d364621817f40b0d9aed2 in 1404ms, sequenceid=65, compaction requested=false 2024-11-26T14:15:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:02,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T14:15:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T14:15:02,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:15:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:02,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:02,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T14:15:02,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7180 sec 2024-11-26T14:15:02,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/6ffed9d952744ffaa46b6be10355d783 is 50, key is test_row_0/A:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:02,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.7550 sec 2024-11-26T14:15:02,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741852_1028 (size=16681) 2024-11-26T14:15:02,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/6ffed9d952744ffaa46b6be10355d783 2024-11-26T14:15:02,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c90ea8e2153540c59e48e75a4cde5a44 is 50, key is test_row_0/B:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630562935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630562936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:02,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741853_1029 (size=12001) 2024-11-26T14:15:03,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c90ea8e2153540c59e48e75a4cde5a44 2024-11-26T14:15:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/44206b8e2fad49ffaa45349bc0fd403d is 50, key is test_row_0/C:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630563054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630563055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741854_1030 (size=12001) 2024-11-26T14:15:03,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/44206b8e2fad49ffaa45349bc0fd403d 2024-11-26T14:15:03,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/6ffed9d952744ffaa46b6be10355d783 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783 2024-11-26T14:15:03,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783, entries=250, sequenceid=83, filesize=16.3 K 2024-11-26T14:15:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c90ea8e2153540c59e48e75a4cde5a44 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44 2024-11-26T14:15:03,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44, entries=150, sequenceid=83, filesize=11.7 K 2024-11-26T14:15:03,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/44206b8e2fad49ffaa45349bc0fd403d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d 2024-11-26T14:15:03,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-26T14:15:03,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c77bbfdf456d364621817f40b0d9aed2 in 403ms, sequenceid=83, compaction requested=true 2024-11-26T14:15:03,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:03,183 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:03,188 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:03,188 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:03,188 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:03,189 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:03,189 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6872ae8ff4124a04ad32e19cb12223ef, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=39.8 K 2024-11-26T14:15:03,190 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6872ae8ff4124a04ad32e19cb12223ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499704 2024-11-26T14:15:03,191 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:03,191 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:03,191 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:03,192 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3f4f3666e20e45ce98c503b6418bc4dc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.3 K 2024-11-26T14:15:03,196 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4f3666e20e45ce98c503b6418bc4dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499704 2024-11-26T14:15:03,196 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0039a9fff6f14a0dae51be20dcdf6e6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630500195 2024-11-26T14:15:03,201 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebb16d3338d4865b197e17b06c1ad96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630500195 2024-11-26T14:15:03,201 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ffed9d952744ffaa46b6be10355d783, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501544 2024-11-26T14:15:03,204 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c90ea8e2153540c59e48e75a4cde5a44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501547 2024-11-26T14:15:03,268 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:03,269 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/ff00af8080764170b0c09399f52e7720 is 50, key is test_row_0/A:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:03,289 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:03,290 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/ec2f5d3b3ba24802a7999a765155026d is 50, key is test_row_0/B:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:03,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:15:03,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:03,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:03,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:03,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741856_1032 (size=12207) 2024-11-26T14:15:03,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/a840a8e981c34699a5c9fa02e6da8d13 is 50, key is test_row_0/A:col10/1732630502921/Put/seqid=0 2024-11-26T14:15:03,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741855_1031 (size=12207) 2024-11-26T14:15:03,410 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/ff00af8080764170b0c09399f52e7720 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/ff00af8080764170b0c09399f52e7720 2024-11-26T14:15:03,411 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/ec2f5d3b3ba24802a7999a765155026d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/ec2f5d3b3ba24802a7999a765155026d 2024-11-26T14:15:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741857_1033 (size=12001) 2024-11-26T14:15:03,447 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into ff00af8080764170b0c09399f52e7720(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:03,448 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:03,448 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630503183; duration=0sec 2024-11-26T14:15:03,449 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:03,449 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:03,449 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:03,462 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:03,462 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:03,462 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:03,463 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/798f1a5525794112858730da7b149b03, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.3 K 2024-11-26T14:15:03,464 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798f1a5525794112858730da7b149b03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732630499704 2024-11-26T14:15:03,466 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9429e953358d4d1699bfeca4cf7f6b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630500195 2024-11-26T14:15:03,467 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44206b8e2fad49ffaa45349bc0fd403d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501547 2024-11-26T14:15:03,475 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into ec2f5d3b3ba24802a7999a765155026d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:03,475 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:03,475 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630503185; duration=0sec 2024-11-26T14:15:03,475 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:03,476 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:03,499 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:03,500 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/933cc75b51aa479e90cc5d12d9550ea7 is 50, key is test_row_0/C:col10/1732630501561/Put/seqid=0 2024-11-26T14:15:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630563512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630563516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741858_1034 (size=12207) 2024-11-26T14:15:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630563636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630563636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630563763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,773 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in c77bbfdf456d364621817f40b0d9aed2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630563781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630563775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,795 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in c77bbfdf456d364621817f40b0d9aed2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,796 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in c77bbfdf456d364621817f40b0d9aed2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/a840a8e981c34699a5c9fa02e6da8d13 2024-11-26T14:15:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630563856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630563863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:03,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3cb11089dc6b4f5b99501fc58a8b19ee is 50, key is test_row_0/B:col10/1732630502921/Put/seqid=0 2024-11-26T14:15:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741859_1035 (size=12001) 2024-11-26T14:15:03,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3cb11089dc6b4f5b99501fc58a8b19ee 2024-11-26T14:15:04,036 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/933cc75b51aa479e90cc5d12d9550ea7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/933cc75b51aa479e90cc5d12d9550ea7 2024-11-26T14:15:04,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/24b0e508acd640c0af48e11a17945a0b is 50, key is test_row_0/C:col10/1732630502921/Put/seqid=0 2024-11-26T14:15:04,133 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 933cc75b51aa479e90cc5d12d9550ea7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:04,133 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:04,133 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630503185; duration=0sec 2024-11-26T14:15:04,133 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:04,133 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741860_1036 (size=12001) 2024-11-26T14:15:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630564190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:04,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630564208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T14:15:04,220 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:04,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T14:15:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:04,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:04,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:04,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:04,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:04,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:04,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/24b0e508acd640c0af48e11a17945a0b 2024-11-26T14:15:04,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/a840a8e981c34699a5c9fa02e6da8d13 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13 2024-11-26T14:15:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:04,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13, entries=150, sequenceid=101, filesize=11.7 K 2024-11-26T14:15:04,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3cb11089dc6b4f5b99501fc58a8b19ee as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee 2024-11-26T14:15:04,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee, entries=150, sequenceid=101, filesize=11.7 K 2024-11-26T14:15:04,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/24b0e508acd640c0af48e11a17945a0b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b 2024-11-26T14:15:04,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b, entries=150, sequenceid=101, filesize=11.7 K 2024-11-26T14:15:04,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c77bbfdf456d364621817f40b0d9aed2 in 1335ms, sequenceid=101, compaction requested=false 2024-11-26T14:15:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:04,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:04,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3df56bd335fd4fa49acec3d2a33e886c is 50, key is test_row_0/A:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:04,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:04,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:04,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741861_1037 (size=12001) 2024-11-26T14:15:04,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3df56bd335fd4fa49acec3d2a33e886c 2024-11-26T14:15:04,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630564854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630564858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:04,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/b4782cb899ec4e8dbe2153588759e65a is 50, key is test_row_0/B:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:04,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741862_1038 (size=12001) 2024-11-26T14:15:04,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/b4782cb899ec4e8dbe2153588759e65a 2024-11-26T14:15:04,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:04,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:04,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0319cf44ab86494ea26ed57cd101c0a7 is 50, key is test_row_0/C:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:04,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:04,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630564964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630564964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741863_1039 (size=12001) 2024-11-26T14:15:05,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:05,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630565183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630565184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0319cf44ab86494ea26ed57cd101c0a7 2024-11-26T14:15:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:05,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:05,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:05,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3df56bd335fd4fa49acec3d2a33e886c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c 2024-11-26T14:15:05,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630565494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630565494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-26T14:15:05,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/b4782cb899ec4e8dbe2153588759e65a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a 2024-11-26T14:15:05,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-26T14:15:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0319cf44ab86494ea26ed57cd101c0a7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7 2024-11-26T14:15:05,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:05,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:05,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7, entries=150, sequenceid=119, filesize=11.7 K 2024-11-26T14:15:05,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c77bbfdf456d364621817f40b0d9aed2 in 905ms, sequenceid=119, compaction requested=true 2024-11-26T14:15:05,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:05,631 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:05,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:05,633 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:05,641 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:05,641 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:05,641 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:05,641 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,641 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/ec2f5d3b3ba24802a7999a765155026d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.4 K 2024-11-26T14:15:05,641 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:05,641 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,642 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/ff00af8080764170b0c09399f52e7720, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.4 K 2024-11-26T14:15:05,642 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2f5d3b3ba24802a7999a765155026d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501547 2024-11-26T14:15:05,643 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff00af8080764170b0c09399f52e7720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501547 2024-11-26T14:15:05,644 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb11089dc6b4f5b99501fc58a8b19ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732630502878 2024-11-26T14:15:05,644 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a840a8e981c34699a5c9fa02e6da8d13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732630502878 2024-11-26T14:15:05,650 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b4782cb899ec4e8dbe2153588759e65a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:05,651 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3df56bd335fd4fa49acec3d2a33e886c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:05,728 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:05,729 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/d118a1ec9ebc408a9edfbea6e4d434a6 is 50, key is test_row_0/A:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:05,746 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#28 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:05,747 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3e804e5541bf421aad6d77d40bf203cf is 50, key is test_row_0/B:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:05,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T14:15:05,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,778 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:05,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/c5ea7e796fe942fd889f00d6df22ba4a is 50, key is test_row_0/A:col10/1732630504814/Put/seqid=0 2024-11-26T14:15:05,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:05,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741865_1041 (size=12309) 2024-11-26T14:15:05,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741864_1040 (size=12309) 2024-11-26T14:15:05,844 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/3e804e5541bf421aad6d77d40bf203cf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3e804e5541bf421aad6d77d40bf203cf 2024-11-26T14:15:05,856 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into 3e804e5541bf421aad6d77d40bf203cf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:05,856 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:05,857 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630505631; duration=0sec 2024-11-26T14:15:05,857 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:05,857 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:05,857 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:05,859 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:05,859 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:05,859 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:05,859 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/933cc75b51aa479e90cc5d12d9550ea7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.4 K 2024-11-26T14:15:05,862 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 933cc75b51aa479e90cc5d12d9550ea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630501547 2024-11-26T14:15:05,863 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b0e508acd640c0af48e11a17945a0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732630502878 2024-11-26T14:15:05,867 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0319cf44ab86494ea26ed57cd101c0a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:05,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741866_1042 (size=12151) 2024-11-26T14:15:05,887 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/c5ea7e796fe942fd889f00d6df22ba4a 2024-11-26T14:15:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630565884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630565888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630565897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:05,915 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:05,915 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/89f346745ef64fd0ac87cc98bf5f9b2c is 50, key is test_row_0/C:col10/1732630504717/Put/seqid=0 2024-11-26T14:15:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/677baba90c354d34918009205eb14f05 is 50, key is test_row_0/B:col10/1732630504814/Put/seqid=0 2024-11-26T14:15:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741867_1043 (size=12309) 2024-11-26T14:15:05,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741868_1044 (size=12151) 2024-11-26T14:15:06,000 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/677baba90c354d34918009205eb14f05 2024-11-26T14:15:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630566003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630566004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630566016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630566017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630566017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/d5762320944241dcbf8d6b3edb1fa77b is 50, key is test_row_0/C:col10/1732630504814/Put/seqid=0 2024-11-26T14:15:06,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741869_1045 (size=12151) 2024-11-26T14:15:06,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630566233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630566233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630566234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,277 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/d118a1ec9ebc408a9edfbea6e4d434a6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d118a1ec9ebc408a9edfbea6e4d434a6 2024-11-26T14:15:06,302 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into d118a1ec9ebc408a9edfbea6e4d434a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:06,302 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:06,302 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630505631; duration=0sec 2024-11-26T14:15:06,304 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:06,304 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:06,395 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/89f346745ef64fd0ac87cc98bf5f9b2c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/89f346745ef64fd0ac87cc98bf5f9b2c 2024-11-26T14:15:06,415 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 89f346745ef64fd0ac87cc98bf5f9b2c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:06,415 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:06,415 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630505631; duration=0sec 2024-11-26T14:15:06,416 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:06,416 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:06,509 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/d5762320944241dcbf8d6b3edb1fa77b 2024-11-26T14:15:06,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630566556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/c5ea7e796fe942fd889f00d6df22ba4a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a 2024-11-26T14:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630566556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630566563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:06,593 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a, entries=150, sequenceid=136, filesize=11.9 K 2024-11-26T14:15:06,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/677baba90c354d34918009205eb14f05 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05 2024-11-26T14:15:06,608 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05, entries=150, sequenceid=136, filesize=11.9 K 2024-11-26T14:15:06,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/d5762320944241dcbf8d6b3edb1fa77b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b 2024-11-26T14:15:06,628 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b, entries=150, sequenceid=136, filesize=11.9 K 2024-11-26T14:15:06,630 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c77bbfdf456d364621817f40b0d9aed2 in 851ms, sequenceid=136, compaction requested=false 2024-11-26T14:15:06,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:06,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:06,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T14:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T14:15:06,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T14:15:06,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3800 sec 2024-11-26T14:15:06,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4590 sec 2024-11-26T14:15:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:07,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:07,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/1d4350e2b987409cb176d272229dbb7b is 50, key is test_row_0/A:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741870_1046 (size=12151) 2024-11-26T14:15:07,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/1d4350e2b987409cb176d272229dbb7b 2024-11-26T14:15:07,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/fbc3c621cae74561a2bd255d1fbcb4e3 is 50, key is test_row_0/B:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:07,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630567121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630567128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630567141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741871_1047 (size=12151) 2024-11-26T14:15:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630567160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/fbc3c621cae74561a2bd255d1fbcb4e3 2024-11-26T14:15:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630567157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/38df7549e3a846bfaa97f379bdfa8da4 is 50, key is test_row_0/C:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:07,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630567264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741872_1048 (size=12151) 2024-11-26T14:15:07,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630567269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630567273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630567294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630567306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630567484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630567484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630567496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630567517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630567537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/38df7549e3a846bfaa97f379bdfa8da4 2024-11-26T14:15:07,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/1d4350e2b987409cb176d272229dbb7b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b 2024-11-26T14:15:07,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:07,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/fbc3c621cae74561a2bd255d1fbcb4e3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3 2024-11-26T14:15:07,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:07,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/38df7549e3a846bfaa97f379bdfa8da4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4 2024-11-26T14:15:07,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630567793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c77bbfdf456d364621817f40b0d9aed2 in 749ms, sequenceid=156, compaction requested=true 2024-11-26T14:15:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T14:15:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:07,806 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:07,810 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:07,810 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:07,810 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:07,811 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:07,811 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d118a1ec9ebc408a9edfbea6e4d434a6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.8 K 2024-11-26T14:15:07,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:07,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:07,814 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d118a1ec9ebc408a9edfbea6e4d434a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:07,824 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ea7e796fe942fd889f00d6df22ba4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732630504814 2024-11-26T14:15:07,825 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:07,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:07,826 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:07,826 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/89f346745ef64fd0ac87cc98bf5f9b2c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.8 K 2024-11-26T14:15:07,826 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4350e2b987409cb176d272229dbb7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:07,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f346745ef64fd0ac87cc98bf5f9b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:07,828 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5762320944241dcbf8d6b3edb1fa77b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732630504814 2024-11-26T14:15:07,829 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38df7549e3a846bfaa97f379bdfa8da4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:07,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/d38a5f3caa1c4adca2553796ed005939 is 50, key is test_row_0/A:col10/1732630507804/Put/seqid=0 2024-11-26T14:15:07,875 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:07,875 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1768c1ffb6864364a2e0658bf2ba633b is 50, key is test_row_0/C:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741873_1049 (size=12151) 2024-11-26T14:15:07,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/d38a5f3caa1c4adca2553796ed005939 2024-11-26T14:15:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630567880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630567860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630567889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,904 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:07,904 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/7e82e468acc64a13a57d3b75162420c7 is 50, key is test_row_0/A:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630567896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:07,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:07,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741874_1050 (size=12561) 2024-11-26T14:15:07,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741875_1051 (size=12561) 2024-11-26T14:15:07,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/583ffb2271ea4f44a3e952a3a49d6999 is 50, key is test_row_0/B:col10/1732630507804/Put/seqid=0 2024-11-26T14:15:07,978 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1768c1ffb6864364a2e0658bf2ba633b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1768c1ffb6864364a2e0658bf2ba633b 2024-11-26T14:15:07,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741876_1052 (size=12151) 2024-11-26T14:15:07,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/583ffb2271ea4f44a3e952a3a49d6999 2024-11-26T14:15:07,997 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/7e82e468acc64a13a57d3b75162420c7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7e82e468acc64a13a57d3b75162420c7 2024-11-26T14:15:08,001 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 1768c1ffb6864364a2e0658bf2ba633b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:08,001 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:08,001 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630507806; duration=0sec 2024-11-26T14:15:08,002 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:08,002 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:08,002 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:08,007 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:08,008 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:08,008 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,008 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3e804e5541bf421aad6d77d40bf203cf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=35.8 K 2024-11-26T14:15:08,010 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e804e5541bf421aad6d77d40bf203cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732630503473 2024-11-26T14:15:08,011 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677baba90c354d34918009205eb14f05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732630504814 2024-11-26T14:15:08,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/17214a2902b04d3f8ca5e97e7a15292c is 50, key is test_row_0/C:col10/1732630507804/Put/seqid=0 2024-11-26T14:15:08,013 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc3c621cae74561a2bd255d1fbcb4e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630568004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630568004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630568013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630568023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,035 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 7e82e468acc64a13a57d3b75162420c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:08,035 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:08,036 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630507806; duration=0sec 2024-11-26T14:15:08,036 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:08,036 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:08,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741877_1053 (size=12151) 2024-11-26T14:15:08,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/17214a2902b04d3f8ca5e97e7a15292c 2024-11-26T14:15:08,060 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:08,061 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/f5fa0346f91f438da5751eecabab9380 is 50, key is test_row_0/B:col10/1732630505884/Put/seqid=0 2024-11-26T14:15:08,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/d38a5f3caa1c4adca2553796ed005939 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939 2024-11-26T14:15:08,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T14:15:08,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/583ffb2271ea4f44a3e952a3a49d6999 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999 2024-11-26T14:15:08,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T14:15:08,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/17214a2902b04d3f8ca5e97e7a15292c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c 2024-11-26T14:15:08,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741878_1054 (size=12561) 2024-11-26T14:15:08,123 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/f5fa0346f91f438da5751eecabab9380 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/f5fa0346f91f438da5751eecabab9380 2024-11-26T14:15:08,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T14:15:08,149 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into f5fa0346f91f438da5751eecabab9380(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:08,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c77bbfdf456d364621817f40b0d9aed2 in 337ms, sequenceid=173, compaction requested=false 2024-11-26T14:15:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:08,149 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:08,150 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630507806; duration=0sec 2024-11-26T14:15:08,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:08,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:08,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:08,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:08,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/26ebdd11fc67474eaccd1685402998f4 is 50, key is test_row_0/A:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630568276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630568280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630568282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630568286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741879_1055 (size=14541) 2024-11-26T14:15:08,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630568316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630568396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630568409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630568408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630568416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T14:15:08,454 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:08,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T14:15:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:08,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:08,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:08,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:08,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630568613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630568624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630568624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630568636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/26ebdd11fc67474eaccd1685402998f4 2024-11-26T14:15:08,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8e3f84a16bd64aaab00fdaa0716afd00 is 50, key is test_row_0/B:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741880_1056 (size=12151) 2024-11-26T14:15:08,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8e3f84a16bd64aaab00fdaa0716afd00 2024-11-26T14:15:08,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/18fbc8f28966415aa6a24a522a48df61 is 50, key is test_row_0/C:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:08,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741881_1057 (size=12151) 2024-11-26T14:15:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630568934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630568936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:08,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:08,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630568944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630568943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:09,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/18fbc8f28966415aa6a24a522a48df61 2024-11-26T14:15:09,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/26ebdd11fc67474eaccd1685402998f4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4 2024-11-26T14:15:09,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4, entries=200, sequenceid=193, filesize=14.2 K 2024-11-26T14:15:09,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8e3f84a16bd64aaab00fdaa0716afd00 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00 2024-11-26T14:15:09,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T14:15:09,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/18fbc8f28966415aa6a24a522a48df61 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61 2024-11-26T14:15:09,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T14:15:09,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c77bbfdf456d364621817f40b0d9aed2 in 1063ms, sequenceid=193, compaction requested=true 2024-11-26T14:15:09,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:09,297 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:09,297 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:09,299 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:09,299 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:09,299 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:09,299 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:09,299 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,299 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/f5fa0346f91f438da5751eecabab9380, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=36.0 K 2024-11-26T14:15:09,299 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,300 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7e82e468acc64a13a57d3b75162420c7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=38.3 K 2024-11-26T14:15:09,301 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fa0346f91f438da5751eecabab9380, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:09,301 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e82e468acc64a13a57d3b75162420c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:09,301 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 583ffb2271ea4f44a3e952a3a49d6999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732630507115 2024-11-26T14:15:09,302 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38a5f3caa1c4adca2553796ed005939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732630507115 2024-11-26T14:15:09,302 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3f84a16bd64aaab00fdaa0716afd00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:09,303 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ebdd11fc67474eaccd1685402998f4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:09,327 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:09,327 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/db1d44ebbbf14ea8a7688539274f024b is 50, key is test_row_0/B:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:09,335 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:09,335 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/097edb0379f5444788402a58390826ed is 50, key is test_row_0/A:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:09,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:09,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:09,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741882_1058 (size=12663) 2024-11-26T14:15:09,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/81c8071e147146cab8404fddf1ce407c is 50, key is test_row_0/A:col10/1732630509356/Put/seqid=0 2024-11-26T14:15:09,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741883_1059 (size=12663) 2024-11-26T14:15:09,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741884_1060 (size=14541) 2024-11-26T14:15:09,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/81c8071e147146cab8404fddf1ce407c 2024-11-26T14:15:09,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9aa03e94c0e14a29b23d033996e573ea is 50, key is test_row_0/B:col10/1732630509356/Put/seqid=0 2024-11-26T14:15:09,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630569453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630569454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630569457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741885_1061 (size=12151) 2024-11-26T14:15:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630569460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630569463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630569564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630569568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630569576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:09,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630569774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630569783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,788 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/db1d44ebbbf14ea8a7688539274f024b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/db1d44ebbbf14ea8a7688539274f024b 2024-11-26T14:15:09,795 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/097edb0379f5444788402a58390826ed as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/097edb0379f5444788402a58390826ed 2024-11-26T14:15:09,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630569796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:09,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:09,804 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into db1d44ebbbf14ea8a7688539274f024b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:09,804 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:09,804 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630509297; duration=0sec 2024-11-26T14:15:09,804 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:09,804 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:09,804 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:09,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:09,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:09,817 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,817 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1768c1ffb6864364a2e0658bf2ba633b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=36.0 K 2024-11-26T14:15:09,817 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 097edb0379f5444788402a58390826ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:09,817 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:09,817 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630509297; duration=0sec 2024-11-26T14:15:09,818 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:09,818 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:09,818 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1768c1ffb6864364a2e0658bf2ba633b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630505842 2024-11-26T14:15:09,819 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 17214a2902b04d3f8ca5e97e7a15292c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732630507115 2024-11-26T14:15:09,820 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 18fbc8f28966415aa6a24a522a48df61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:09,863 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#49 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:09,863 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/b3a408803bea4986a477d4ba9f8c1480 is 50, key is test_row_0/C:col10/1732630508232/Put/seqid=0 2024-11-26T14:15:09,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9aa03e94c0e14a29b23d033996e573ea 2024-11-26T14:15:09,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:09,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:09,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:09,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:09,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741886_1062 (size=12663) 2024-11-26T14:15:09,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/5008dcb8a1d741529135191901d50090 is 50, key is test_row_0/C:col10/1732630509356/Put/seqid=0 2024-11-26T14:15:09,910 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/b3a408803bea4986a477d4ba9f8c1480 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/b3a408803bea4986a477d4ba9f8c1480 2024-11-26T14:15:09,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741887_1063 (size=12151) 2024-11-26T14:15:09,923 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into b3a408803bea4986a477d4ba9f8c1480(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:09,923 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:09,923 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630509297; duration=0sec 2024-11-26T14:15:09,923 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:09,923 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:10,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:10,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630570084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630570093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630570113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:10,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:10,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/5008dcb8a1d741529135191901d50090 2024-11-26T14:15:10,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/81c8071e147146cab8404fddf1ce407c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c 2024-11-26T14:15:10,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c, entries=200, sequenceid=209, filesize=14.2 K 2024-11-26T14:15:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/9aa03e94c0e14a29b23d033996e573ea as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea 2024-11-26T14:15:10,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea, entries=150, sequenceid=209, filesize=11.9 K 2024-11-26T14:15:10,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/5008dcb8a1d741529135191901d50090 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090 2024-11-26T14:15:10,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090, entries=150, sequenceid=209, filesize=11.9 K 2024-11-26T14:15:10,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c77bbfdf456d364621817f40b0d9aed2 in 1000ms, sequenceid=209, compaction requested=false 2024-11-26T14:15:10,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:10,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T14:15:10,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:10,362 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:10,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/924a1d2797574cfdb55db6b6c7520d9b is 50, key is test_row_0/A:col10/1732630509453/Put/seqid=0 2024-11-26T14:15:10,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741888_1064 (size=12151) 2024-11-26T14:15:10,390 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/924a1d2797574cfdb55db6b6c7520d9b 2024-11-26T14:15:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/a3cd0faeffe341f5abc2dd9d68faf334 is 50, key is test_row_0/B:col10/1732630509453/Put/seqid=0 2024-11-26T14:15:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741889_1065 (size=12151) 2024-11-26T14:15:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:10,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:10,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630570508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630570510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630570603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630570603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630570623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630570623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:10,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630570634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,811 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/a3cd0faeffe341f5abc2dd9d68faf334 2024-11-26T14:15:10,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6c593facc1741449bb97b23117e853e is 50, key is test_row_0/C:col10/1732630509453/Put/seqid=0 2024-11-26T14:15:10,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741890_1066 (size=12151) 2024-11-26T14:15:10,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630570833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630570833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630571143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630571143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,231 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6c593facc1741449bb97b23117e853e 2024-11-26T14:15:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/924a1d2797574cfdb55db6b6c7520d9b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b 2024-11-26T14:15:11,268 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b, entries=150, sequenceid=228, filesize=11.9 K 2024-11-26T14:15:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/a3cd0faeffe341f5abc2dd9d68faf334 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334 2024-11-26T14:15:11,279 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334, entries=150, sequenceid=228, filesize=11.9 K 2024-11-26T14:15:11,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6c593facc1741449bb97b23117e853e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e 2024-11-26T14:15:11,288 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e, entries=150, sequenceid=228, filesize=11.9 K 2024-11-26T14:15:11,297 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c77bbfdf456d364621817f40b0d9aed2 in 936ms, sequenceid=228, compaction requested=true 2024-11-26T14:15:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T14:15:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T14:15:11,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T14:15:11,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8360 sec 2024-11-26T14:15:11,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8580 sec 2024-11-26T14:15:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:11,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:11,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:11,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/7b9ed0b1dd824b27a285d58abd15623b is 50, key is test_row_0/A:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:11,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741891_1067 (size=14541) 2024-11-26T14:15:11,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/7b9ed0b1dd824b27a285d58abd15623b 2024-11-26T14:15:11,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630571663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630571667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630571668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630571670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630571670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/0322071bcf27484eac55a4fd5f25be07 is 50, key is test_row_0/B:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741892_1068 (size=12151) 2024-11-26T14:15:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630571774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630571785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630571785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630571787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630571787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630571983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630571994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630571994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630572003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630572003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/0322071bcf27484eac55a4fd5f25be07 2024-11-26T14:15:12,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0ce7d68dbb36475c815f080dc15f41ae is 50, key is test_row_0/C:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:12,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741893_1069 (size=12151) 2024-11-26T14:15:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630572296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630572304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630572305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630572315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630572316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0ce7d68dbb36475c815f080dc15f41ae 2024-11-26T14:15:12,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/7b9ed0b1dd824b27a285d58abd15623b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b 2024-11-26T14:15:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T14:15:12,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:12,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T14:15:12,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:12,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:12,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:12,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b, entries=200, sequenceid=245, filesize=14.2 K 2024-11-26T14:15:12,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/0322071bcf27484eac55a4fd5f25be07 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07 2024-11-26T14:15:12,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07, entries=150, sequenceid=245, filesize=11.9 K 2024-11-26T14:15:12,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/0ce7d68dbb36475c815f080dc15f41ae as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae 2024-11-26T14:15:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:12,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae, entries=150, sequenceid=245, filesize=11.9 K 2024-11-26T14:15:12,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c77bbfdf456d364621817f40b0d9aed2 in 1170ms, sequenceid=245, compaction requested=true 2024-11-26T14:15:12,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:12,787 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:12,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:12,789 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:12,792 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:12,792 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:12,792 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,792 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/097edb0379f5444788402a58390826ed, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=52.6 K 2024-11-26T14:15:12,793 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 097edb0379f5444788402a58390826ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:12,794 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c8071e147146cab8404fddf1ce407c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732630508262 2024-11-26T14:15:12,794 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 924a1d2797574cfdb55db6b6c7520d9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732630509450 2024-11-26T14:15:12,795 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9ed0b1dd824b27a285d58abd15623b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:12,808 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:12,808 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:12,808 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,809 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/db1d44ebbbf14ea8a7688539274f024b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=48.0 K 2024-11-26T14:15:12,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:12,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:12,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:12,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:12,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:12,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:12,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:12,812 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting db1d44ebbbf14ea8a7688539274f024b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:12,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa03e94c0e14a29b23d033996e573ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732630508262 2024-11-26T14:15:12,821 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a3cd0faeffe341f5abc2dd9d68faf334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732630509450 2024-11-26T14:15:12,828 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0322071bcf27484eac55a4fd5f25be07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:12,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/816cdc6587f7414f9bcfc646fac28001 is 50, key is test_row_0/A:col10/1732630512806/Put/seqid=0 2024-11-26T14:15:12,829 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#57 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:12,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:12,831 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/da9df16d6fe945ab858f8002c1b9d2ac is 50, key is test_row_0/A:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:12,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741895_1071 (size=12799) 2024-11-26T14:15:12,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630572862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630572872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630572875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630572877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630572860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741894_1070 (size=14691) 2024-11-26T14:15:12,903 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:12,904 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf is 50, key is test_row_0/B:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:12,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741896_1072 (size=12799) 2024-11-26T14:15:12,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:12,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:12,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630572983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,013 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf 2024-11-26T14:15:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,065 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into 88b21d0e13bf4f8ea7b6d14d98cfb9bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:13,065 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:13,065 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=12, startTime=1732630512789; duration=0sec 2024-11-26T14:15:13,065 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:13,065 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:13,065 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:13,071 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:13,071 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:13,071 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,071 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/b3a408803bea4986a477d4ba9f8c1480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=48.0 K 2024-11-26T14:15:13,072 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a408803bea4986a477d4ba9f8c1480, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732630507858 2024-11-26T14:15:13,072 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5008dcb8a1d741529135191901d50090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732630508262 2024-11-26T14:15:13,076 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c593facc1741449bb97b23117e853e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732630509450 2024-11-26T14:15:13,076 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce7d68dbb36475c815f080dc15f41ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:13,095 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:13,099 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/7bc4f8626a3f41bbabbaa2cb07c87c82 is 50, key is test_row_0/C:col10/1732630510491/Put/seqid=0 2024-11-26T14:15:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741897_1073 (size=12799) 2024-11-26T14:15:13,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,158 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/7bc4f8626a3f41bbabbaa2cb07c87c82 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/7bc4f8626a3f41bbabbaa2cb07c87c82 2024-11-26T14:15:13,176 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 7bc4f8626a3f41bbabbaa2cb07c87c82(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:13,176 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:13,177 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=12, startTime=1732630512789; duration=0sec 2024-11-26T14:15:13,177 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:13,177 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630573203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630573213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630573213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630573225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630573225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/816cdc6587f7414f9bcfc646fac28001 2024-11-26T14:15:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:13,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,326 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/da9df16d6fe945ab858f8002c1b9d2ac as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/da9df16d6fe945ab858f8002c1b9d2ac 2024-11-26T14:15:13,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/52a4f0dbe2b4428ebd23acd79ed617d5 is 50, key is test_row_0/B:col10/1732630512806/Put/seqid=0 2024-11-26T14:15:13,346 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into da9df16d6fe945ab858f8002c1b9d2ac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:13,346 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:13,346 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=12, startTime=1732630512787; duration=0sec 2024-11-26T14:15:13,346 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:13,346 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:13,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741898_1074 (size=12301) 2024-11-26T14:15:13,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/52a4f0dbe2b4428ebd23acd79ed617d5 2024-11-26T14:15:13,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/732e1dd0ff7246b28bbe4e55433bf46c is 50, key is test_row_0/C:col10/1732630512806/Put/seqid=0 2024-11-26T14:15:13,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741899_1075 (size=12301) 2024-11-26T14:15:13,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/732e1dd0ff7246b28bbe4e55433bf46c 2024-11-26T14:15:13,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/816cdc6587f7414f9bcfc646fac28001 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001 2024-11-26T14:15:13,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001, entries=200, sequenceid=262, filesize=14.3 K 2024-11-26T14:15:13,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/52a4f0dbe2b4428ebd23acd79ed617d5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5 2024-11-26T14:15:13,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T14:15:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/732e1dd0ff7246b28bbe4e55433bf46c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c 2024-11-26T14:15:13,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T14:15:13,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c77bbfdf456d364621817f40b0d9aed2 in 717ms, sequenceid=262, compaction requested=false 2024-11-26T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:13,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:13,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:13,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:13,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/bca7b22e19c447f59e0559d7ddbd92b8 is 50, key is test_row_0/A:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:13,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741900_1076 (size=14741) 2024-11-26T14:15:13,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/bca7b22e19c447f59e0559d7ddbd92b8 2024-11-26T14:15:13,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/85bc205660d64aff9e2a87948bfaf163 is 50, key is test_row_0/B:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:13,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741901_1077 (size=12301) 2024-11-26T14:15:13,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/85bc205660d64aff9e2a87948bfaf163 2024-11-26T14:15:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630573673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630573682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630573682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/c38799e3c8b349f6a799bfcb6383a09b is 50, key is test_row_0/C:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630573690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630573697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741902_1078 (size=12301) 2024-11-26T14:15:13,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/c38799e3c8b349f6a799bfcb6383a09b 2024-11-26T14:15:13,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/bca7b22e19c447f59e0559d7ddbd92b8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8 2024-11-26T14:15:13,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8, entries=200, sequenceid=281, filesize=14.4 K 2024-11-26T14:15:13,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/85bc205660d64aff9e2a87948bfaf163 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163 2024-11-26T14:15:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163, entries=150, sequenceid=281, filesize=12.0 K 2024-11-26T14:15:13,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/c38799e3c8b349f6a799bfcb6383a09b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b 2024-11-26T14:15:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630573803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:13,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b, entries=150, sequenceid=281, filesize=12.0 K 2024-11-26T14:15:13,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c77bbfdf456d364621817f40b0d9aed2 in 290ms, sequenceid=281, compaction requested=true 2024-11-26T14:15:13,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:13,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:13,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:13,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:13,826 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,826 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/da9df16d6fe945ab858f8002c1b9d2ac, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=41.2 K 2024-11-26T14:15:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:13,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:13,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9df16d6fe945ab858f8002c1b9d2ac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:13,827 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:13,828 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 816cdc6587f7414f9bcfc646fac28001, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732630511654 2024-11-26T14:15:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:13,829 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting bca7b22e19c447f59e0559d7ddbd92b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512842 2024-11-26T14:15:13,829 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:13,830 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:13,830 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,830 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=36.5 K 2024-11-26T14:15:13,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:13,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:13,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:13,832 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b21d0e13bf4f8ea7b6d14d98cfb9bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:13,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:13,832 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a4f0dbe2b4428ebd23acd79ed617d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732630511665 2024-11-26T14:15:13,833 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bc205660d64aff9e2a87948bfaf163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512859 2024-11-26T14:15:13,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:15:13,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:13,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:13,850 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:13,851 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/b685f674aaba43b7ba876c3cf81afb81 is 50, key is test_row_0/B:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:13,853 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:13,854 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/20fd084d583f4dc0a8ff94a484e46a87 is 50, key is test_row_0/A:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:13,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/259455fe7f704463ae8cbc6e0bece095 is 50, key is test_row_0/A:col10/1732630513681/Put/seqid=0 2024-11-26T14:15:13,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741904_1080 (size=13051) 2024-11-26T14:15:13,927 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/20fd084d583f4dc0a8ff94a484e46a87 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/20fd084d583f4dc0a8ff94a484e46a87 2024-11-26T14:15:13,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741903_1079 (size=13051) 2024-11-26T14:15:13,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:13,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:13,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:13,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741905_1081 (size=19621) 2024-11-26T14:15:13,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/259455fe7f704463ae8cbc6e0bece095 2024-11-26T14:15:13,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/4ffa0d8e8b4649a8ae53d2e3e334db76 is 50, key is test_row_0/B:col10/1732630513681/Put/seqid=0 2024-11-26T14:15:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630573891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,008 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 20fd084d583f4dc0a8ff94a484e46a87(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:14,009 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:14,009 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630513824; duration=0sec 2024-11-26T14:15:14,009 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:14,009 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:14,009 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:14,011 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:14,011 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:14,011 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,011 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/7bc4f8626a3f41bbabbaa2cb07c87c82, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=36.5 K 2024-11-26T14:15:14,012 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc4f8626a3f41bbabbaa2cb07c87c82, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732630510491 2024-11-26T14:15:14,013 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732e1dd0ff7246b28bbe4e55433bf46c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732630511665 2024-11-26T14:15:14,014 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38799e3c8b349f6a799bfcb6383a09b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512859 2024-11-26T14:15:14,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630573996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741906_1082 (size=12301) 2024-11-26T14:15:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/4ffa0d8e8b4649a8ae53d2e3e334db76 2024-11-26T14:15:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630573985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,039 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:14,040 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/aed9ae38ebbd440e864058d5a207e905 is 50, key is test_row_0/C:col10/1732630513528/Put/seqid=0 2024-11-26T14:15:14,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/e6c57d261dc94f98920c270905ccbdf7 is 50, key is test_row_0/C:col10/1732630513681/Put/seqid=0 2024-11-26T14:15:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630574023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741907_1083 (size=13051) 2024-11-26T14:15:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741908_1084 (size=12301) 2024-11-26T14:15:14,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/aed9ae38ebbd440e864058d5a207e905 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/aed9ae38ebbd440e864058d5a207e905 2024-11-26T14:15:14,091 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into aed9ae38ebbd440e864058d5a207e905(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:14,091 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:14,091 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=13, startTime=1732630513831; duration=0sec 2024-11-26T14:15:14,093 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:14,093 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:14,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630574103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630574123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630574134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630574323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630574333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,347 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/b685f674aaba43b7ba876c3cf81afb81 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b685f674aaba43b7ba876c3cf81afb81 2024-11-26T14:15:14,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630574344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,357 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into b685f674aaba43b7ba876c3cf81afb81(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:14,357 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:14,357 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630513827; duration=0sec 2024-11-26T14:15:14,357 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:14,357 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:14,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630574353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:14,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/e6c57d261dc94f98920c270905ccbdf7 2024-11-26T14:15:14,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/259455fe7f704463ae8cbc6e0bece095 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095 2024-11-26T14:15:14,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095, entries=300, sequenceid=300, filesize=19.2 K 2024-11-26T14:15:14,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/4ffa0d8e8b4649a8ae53d2e3e334db76 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76 2024-11-26T14:15:14,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76, entries=150, sequenceid=300, filesize=12.0 K 2024-11-26T14:15:14,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/e6c57d261dc94f98920c270905ccbdf7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7 2024-11-26T14:15:14,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7, entries=150, sequenceid=300, filesize=12.0 K 2024-11-26T14:15:14,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c77bbfdf456d364621817f40b0d9aed2 in 679ms, sequenceid=300, compaction requested=false 2024-11-26T14:15:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:14,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T14:15:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:14,577 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T14:15:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/45aa222890ca4866aed75ccae920dfcd is 50, key is test_row_0/A:col10/1732630513911/Put/seqid=0 2024-11-26T14:15:14,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741909_1085 (size=12301) 2024-11-26T14:15:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:14,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:14,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630574669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630574671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630574672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630574784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630574784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630574785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630574873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630574993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630574993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630574994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630574994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/45aa222890ca4866aed75ccae920dfcd 2024-11-26T14:15:15,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c018c3bc9bd04de5b3d4bc7274deca56 is 50, key is test_row_0/B:col10/1732630513911/Put/seqid=0 2024-11-26T14:15:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741910_1086 (size=12301) 2024-11-26T14:15:15,035 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c018c3bc9bd04de5b3d4bc7274deca56 2024-11-26T14:15:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/267fa12ff4ce486eb8b62bd0cc56cf22 is 50, key is test_row_0/C:col10/1732630513911/Put/seqid=0 2024-11-26T14:15:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741911_1087 (size=12301) 2024-11-26T14:15:15,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630575303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630575303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630575303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630575303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,458 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/267fa12ff4ce486eb8b62bd0cc56cf22 2024-11-26T14:15:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/45aa222890ca4866aed75ccae920dfcd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd 2024-11-26T14:15:15,470 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd, entries=150, sequenceid=317, filesize=12.0 K 2024-11-26T14:15:15,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c018c3bc9bd04de5b3d4bc7274deca56 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56 2024-11-26T14:15:15,477 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56, entries=150, sequenceid=317, filesize=12.0 K 2024-11-26T14:15:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/267fa12ff4ce486eb8b62bd0cc56cf22 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22 2024-11-26T14:15:15,485 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22, entries=150, sequenceid=317, filesize=12.0 K 2024-11-26T14:15:15,486 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c77bbfdf456d364621817f40b0d9aed2 in 909ms, sequenceid=317, compaction requested=true 2024-11-26T14:15:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T14:15:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T14:15:15,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T14:15:15,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8160 sec 2024-11-26T14:15:15,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.8260 sec 2024-11-26T14:15:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:15,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:15,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f74dc525843d467e8352d0eddb03c5f7 is 50, key is test_row_0/A:col10/1732630515815/Put/seqid=0 2024-11-26T14:15:15,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630575828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630575828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630575828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741912_1088 (size=14741) 2024-11-26T14:15:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f74dc525843d467e8352d0eddb03c5f7 2024-11-26T14:15:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630575830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/119160701a3141eb970fdb0d5f1ed180 is 50, key is test_row_0/B:col10/1732630515815/Put/seqid=0 2024-11-26T14:15:15,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741913_1089 (size=12301) 2024-11-26T14:15:15,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/119160701a3141eb970fdb0d5f1ed180 2024-11-26T14:15:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630575883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,890 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in c77bbfdf456d364621817f40b0d9aed2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6a61d7e198144b5b2eefd142e3a8d1c is 50, key is test_row_0/C:col10/1732630515815/Put/seqid=0 2024-11-26T14:15:15,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741914_1090 (size=12301) 2024-11-26T14:15:15,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630575944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630575944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630575944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630575944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630576153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630576154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630576154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630576156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6a61d7e198144b5b2eefd142e3a8d1c 2024-11-26T14:15:16,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f74dc525843d467e8352d0eddb03c5f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7 2024-11-26T14:15:16,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7, entries=200, sequenceid=336, filesize=14.4 K 2024-11-26T14:15:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/119160701a3141eb970fdb0d5f1ed180 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180 2024-11-26T14:15:16,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180, entries=150, sequenceid=336, filesize=12.0 K 2024-11-26T14:15:16,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/f6a61d7e198144b5b2eefd142e3a8d1c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c 2024-11-26T14:15:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-26T14:15:16,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c77bbfdf456d364621817f40b0d9aed2 in 544ms, sequenceid=336, compaction requested=true 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:16,360 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:16,360 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:16,365 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:16,365 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:16,365 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:16,365 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:16,365 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:16,365 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:16,365 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/20fd084d583f4dc0a8ff94a484e46a87, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=58.3 K 2024-11-26T14:15:16,365 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b685f674aaba43b7ba876c3cf81afb81, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=48.8 K 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b685f674aaba43b7ba876c3cf81afb81, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512859 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20fd084d583f4dc0a8ff94a484e46a87, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512859 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259455fe7f704463ae8cbc6e0bece095, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732630513675 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffa0d8e8b4649a8ae53d2e3e334db76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732630513681 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c018c3bc9bd04de5b3d4bc7274deca56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732630513888 2024-11-26T14:15:16,366 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45aa222890ca4866aed75ccae920dfcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732630513888 2024-11-26T14:15:16,367 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 119160701a3141eb970fdb0d5f1ed180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732630514663 2024-11-26T14:15:16,367 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f74dc525843d467e8352d0eddb03c5f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732630514663 2024-11-26T14:15:16,381 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:16,382 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0f9420157e2143aeacd338c8d63f6032 is 50, key is test_row_0/A:col10/1732630515815/Put/seqid=0 2024-11-26T14:15:16,383 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:16,383 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/2e496b92c7cf480dacca14e61314fd49 is 50, key is test_row_0/B:col10/1732630515815/Put/seqid=0 2024-11-26T14:15:16,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741916_1092 (size=13187) 2024-11-26T14:15:16,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741915_1091 (size=13187) 2024-11-26T14:15:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:16,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:15:16,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f5c8e2510868416bb4247aca99a33761 is 50, key is test_row_0/A:col10/1732630515826/Put/seqid=0 2024-11-26T14:15:16,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741917_1093 (size=12301) 2024-11-26T14:15:16,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f5c8e2510868416bb4247aca99a33761 2024-11-26T14:15:16,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8487af05e1354750ab3f9ceecb656272 is 50, key is test_row_0/B:col10/1732630515826/Put/seqid=0 2024-11-26T14:15:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741918_1094 (size=12301) 2024-11-26T14:15:16,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8487af05e1354750ab3f9ceecb656272 2024-11-26T14:15:16,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630576496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630576499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630576508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a4042b172a344658aac00833d67f6af9 is 50, key is test_row_0/C:col10/1732630515826/Put/seqid=0 2024-11-26T14:15:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630576509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741919_1095 (size=12301) 2024-11-26T14:15:16,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a4042b172a344658aac00833d67f6af9 2024-11-26T14:15:16,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/f5c8e2510868416bb4247aca99a33761 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761 2024-11-26T14:15:16,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761, entries=150, sequenceid=351, filesize=12.0 K 2024-11-26T14:15:16,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/8487af05e1354750ab3f9ceecb656272 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272 2024-11-26T14:15:16,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272, entries=150, sequenceid=351, filesize=12.0 K 2024-11-26T14:15:16,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a4042b172a344658aac00833d67f6af9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9 2024-11-26T14:15:16,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9, entries=150, sequenceid=351, filesize=12.0 K 2024-11-26T14:15:16,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c77bbfdf456d364621817f40b0d9aed2 in 124ms, sequenceid=351, compaction requested=true 2024-11-26T14:15:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T14:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T14:15:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:16,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:15:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:16,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/46dff3df1d3943f7a82f542d158b7425 is 50, key is test_row_0/A:col10/1732630516618/Put/seqid=0 2024-11-26T14:15:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630576640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741920_1096 (size=14741) 2024-11-26T14:15:16,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630576646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630576649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630576652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630576763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630576763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630576764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630576764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,809 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/2e496b92c7cf480dacca14e61314fd49 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/2e496b92c7cf480dacca14e61314fd49 2024-11-26T14:15:16,815 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/0f9420157e2143aeacd338c8d63f6032 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0f9420157e2143aeacd338c8d63f6032 2024-11-26T14:15:16,815 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into 2e496b92c7cf480dacca14e61314fd49(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:16,815 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:16,815 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=12, startTime=1732630516360; duration=0sec 2024-11-26T14:15:16,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T14:15:16,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:16,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-26T14:15:16,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T14:15:16,816 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T14:15:16,817 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. because compaction request was cancelled 2024-11-26T14:15:16,817 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:16,817 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T14:15:16,819 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T14:15:16,819 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/C is initiating minor compaction (all files) 2024-11-26T14:15:16,819 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/C in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:16,820 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/aed9ae38ebbd440e864058d5a207e905, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=60.8 K 2024-11-26T14:15:16,820 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aed9ae38ebbd440e864058d5a207e905, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732630512859 2024-11-26T14:15:16,820 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c57d261dc94f98920c270905ccbdf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732630513681 2024-11-26T14:15:16,822 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 267fa12ff4ce486eb8b62bd0cc56cf22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732630513888 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f6a61d7e198144b5b2eefd142e3a8d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732630514663 2024-11-26T14:15:16,823 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 0f9420157e2143aeacd338c8d63f6032(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:16,823 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=12, startTime=1732630516360; duration=0sec 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a4042b172a344658aac00833d67f6af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732630515822 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:16,823 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-26T14:15:16,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T14:15:16,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T14:15:16,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. because compaction request was cancelled 2024-11-26T14:15:16,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:16,824 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T14:15:16,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T14:15:16,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T14:15:16,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. because compaction request was cancelled 2024-11-26T14:15:16,826 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T14:15:16,844 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:16,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T14:15:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:16,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:16,867 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:16,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:16,877 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:16,878 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/3da22dd715df45968450cf0e4b402bd8 is 50, key is test_row_0/C:col10/1732630515826/Put/seqid=0 2024-11-26T14:15:16,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741921_1097 (size=13221) 2024-11-26T14:15:16,944 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/3da22dd715df45968450cf0e4b402bd8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/3da22dd715df45968450cf0e4b402bd8 2024-11-26T14:15:16,957 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/C of c77bbfdf456d364621817f40b0d9aed2 into 3da22dd715df45968450cf0e4b402bd8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:16,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:16,957 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/C, priority=11, startTime=1732630516593; duration=0sec 2024-11-26T14:15:16,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:16,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:16,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630576973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630576974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630576974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630576976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/46dff3df1d3943f7a82f542d158b7425 2024-11-26T14:15:17,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/93476a97d83743b78e64aed11e21ccb2 is 50, key is test_row_0/B:col10/1732630516618/Put/seqid=0 2024-11-26T14:15:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741922_1098 (size=12301) 2024-11-26T14:15:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:17,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630577294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630577295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630577295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630577295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:17,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/93476a97d83743b78e64aed11e21ccb2 2024-11-26T14:15:17,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a62107d405fb4b1db073466fa3281224 is 50, key is test_row_0/C:col10/1732630516618/Put/seqid=0 2024-11-26T14:15:17,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741923_1099 (size=12301) 2024-11-26T14:15:17,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55202 deadline: 1732630577803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55180 deadline: 1732630577813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55234 deadline: 1732630577813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55178 deadline: 1732630577813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,864 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T14:15:17,864 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:17,866 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T14:15:17,866 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:17,867 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T14:15:17,867 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:17,868 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,868 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,868 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,869 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,869 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,869 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:17,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:17,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:17,871 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T14:15:17,872 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:17,872 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,872 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:17,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55204 deadline: 1732630577913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:17,914 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in c77bbfdf456d364621817f40b0d9aed2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:17,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a62107d405fb4b1db073466fa3281224 2024-11-26T14:15:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/46dff3df1d3943f7a82f542d158b7425 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425 2024-11-26T14:15:17,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425, entries=200, sequenceid=370, filesize=14.4 K 2024-11-26T14:15:17,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/93476a97d83743b78e64aed11e21ccb2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2 2024-11-26T14:15:17,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2, entries=150, sequenceid=370, filesize=12.0 K 2024-11-26T14:15:17,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/a62107d405fb4b1db073466fa3281224 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a62107d405fb4b1db073466fa3281224 2024-11-26T14:15:17,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:17,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a62107d405fb4b1db073466fa3281224, entries=150, sequenceid=370, filesize=12.0 K 2024-11-26T14:15:17,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c77bbfdf456d364621817f40b0d9aed2 in 1338ms, sequenceid=370, compaction requested=true 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:17,961 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:17,961 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c77bbfdf456d364621817f40b0d9aed2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:17,962 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:17,962 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:17,962 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/A is initiating minor compaction (all files) 2024-11-26T14:15:17,962 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): c77bbfdf456d364621817f40b0d9aed2/B is initiating minor compaction (all files) 2024-11-26T14:15:17,962 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/A in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,962 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c77bbfdf456d364621817f40b0d9aed2/B in TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:17,962 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0f9420157e2143aeacd338c8d63f6032, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=39.3 K 2024-11-26T14:15:17,962 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/2e496b92c7cf480dacca14e61314fd49, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp, totalSize=36.9 K 2024-11-26T14:15:17,963 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f9420157e2143aeacd338c8d63f6032, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732630514663 2024-11-26T14:15:17,963 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e496b92c7cf480dacca14e61314fd49, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732630514663 2024-11-26T14:15:17,964 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c8e2510868416bb4247aca99a33761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732630515822 2024-11-26T14:15:17,964 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8487af05e1354750ab3f9ceecb656272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732630515822 2024-11-26T14:15:17,964 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46dff3df1d3943f7a82f542d158b7425, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732630516492 2024-11-26T14:15:17,964 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 93476a97d83743b78e64aed11e21ccb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732630516492 2024-11-26T14:15:17,974 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:17,974 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c77bbfdf456d364621817f40b0d9aed2#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:17,975 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/4a17daebb06a40b0a03d15efc8ed5a86 is 50, key is test_row_0/A:col10/1732630516618/Put/seqid=0 2024-11-26T14:15:17,975 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/da3f448ef3c14ef6aa8d117a46355695 is 50, key is test_row_0/B:col10/1732630516618/Put/seqid=0 2024-11-26T14:15:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741924_1100 (size=13289) 2024-11-26T14:15:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741925_1101 (size=13289) 2024-11-26T14:15:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:18,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T14:15:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:18,108 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3e81752d489b446ea42216f2d158b104 is 50, key is test_row_0/A:col10/1732630516637/Put/seqid=0 2024-11-26T14:15:18,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741926_1102 (size=12301) 2024-11-26T14:15:18,384 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/da3f448ef3c14ef6aa8d117a46355695 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/da3f448ef3c14ef6aa8d117a46355695 2024-11-26T14:15:18,384 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:15:18,388 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/4a17daebb06a40b0a03d15efc8ed5a86 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/4a17daebb06a40b0a03d15efc8ed5a86 2024-11-26T14:15:18,390 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/B of c77bbfdf456d364621817f40b0d9aed2 into da3f448ef3c14ef6aa8d117a46355695(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:18,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:18,390 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/B, priority=13, startTime=1732630517961; duration=0sec 2024-11-26T14:15:18,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:18,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:B 2024-11-26T14:15:18,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T14:15:18,391 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T14:15:18,391 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T14:15:18,391 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. because compaction request was cancelled 2024-11-26T14:15:18,391 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:C 2024-11-26T14:15:18,394 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c77bbfdf456d364621817f40b0d9aed2/A of c77bbfdf456d364621817f40b0d9aed2 into 4a17daebb06a40b0a03d15efc8ed5a86(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:18,394 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:18,394 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., storeName=c77bbfdf456d364621817f40b0d9aed2/A, priority=13, startTime=1732630517961; duration=0sec 2024-11-26T14:15:18,394 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:18,394 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c77bbfdf456d364621817f40b0d9aed2:A 2024-11-26T14:15:18,520 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3e81752d489b446ea42216f2d158b104 2024-11-26T14:15:18,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/46d72f4a99e642e89e32614de693c55c is 50, key is test_row_0/B:col10/1732630516637/Put/seqid=0 2024-11-26T14:15:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741927_1103 (size=12301) 2024-11-26T14:15:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:18,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. as already flushing 2024-11-26T14:15:18,826 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T14:15:18,826 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:18,826 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,826 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:18,827 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T14:15:18,827 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:18,827 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,827 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:18,841 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T14:15:18,841 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:18,841 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,841 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,842 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T14:15:18,842 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:18,842 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,842 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:18,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:18,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:18,936 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/46d72f4a99e642e89e32614de693c55c 2024-11-26T14:15:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/52cb6e96f446444ca8932fa24d806dea is 50, key is test_row_0/C:col10/1732630516637/Put/seqid=0 2024-11-26T14:15:18,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741928_1104 (size=12301) 2024-11-26T14:15:18,984 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/52cb6e96f446444ca8932fa24d806dea 2024-11-26T14:15:19,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/3e81752d489b446ea42216f2d158b104 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3e81752d489b446ea42216f2d158b104 2024-11-26T14:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:19,026 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3e81752d489b446ea42216f2d158b104, entries=150, sequenceid=386, filesize=12.0 K 2024-11-26T14:15:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/46d72f4a99e642e89e32614de693c55c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/46d72f4a99e642e89e32614de693c55c 2024-11-26T14:15:19,068 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/46d72f4a99e642e89e32614de693c55c, entries=150, sequenceid=386, filesize=12.0 K 2024-11-26T14:15:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/52cb6e96f446444ca8932fa24d806dea as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/52cb6e96f446444ca8932fa24d806dea 2024-11-26T14:15:19,090 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/52cb6e96f446444ca8932fa24d806dea, entries=150, sequenceid=386, filesize=12.0 K 2024-11-26T14:15:19,093 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for c77bbfdf456d364621817f40b0d9aed2 in 984ms, sequenceid=386, compaction requested=true 2024-11-26T14:15:19,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c77bbfdf456d364621817f40b0d9aed2: 2024-11-26T14:15:19,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:19,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T14:15:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T14:15:19,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2540 sec 2024-11-26T14:15:19,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T14:15:19,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-26T14:15:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T14:15:21,034 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:21,945 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T14:15:21,945 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:21,946 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:21,946 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:21,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1646 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1607 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 750 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2248 rows 2024-11-26T14:15:21,946 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 749 2024-11-26T14:15:21,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2243 rows 2024-11-26T14:15:21,947 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:15:21,947 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:15:21,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:21,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:21,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:21,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:15:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:21,959 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630521959"}]},"ts":"1732630521959"} 2024-11-26T14:15:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T14:15:21,965 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:15:21,965 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:15:21,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:15:21,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, UNASSIGN}] 2024-11-26T14:15:21,979 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, UNASSIGN 2024-11-26T14:15:21,981 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=c77bbfdf456d364621817f40b0d9aed2, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:21,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, UNASSIGN because future has completed 2024-11-26T14:15:21,992 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:15:21,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T14:15:22,151 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:22,151 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:15:22,152 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing c77bbfdf456d364621817f40b0d9aed2, disabling compactions & flushes 2024-11-26T14:15:22,152 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:22,152 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:22,152 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. after waiting 0 ms 2024-11-26T14:15:22,152 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:22,152 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing c77bbfdf456d364621817f40b0d9aed2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=A 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=B 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c77bbfdf456d364621817f40b0d9aed2, store=C 2024-11-26T14:15:22,153 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:22,158 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 is 50, key is test_row_1/A:col10/1732630518835/Put/seqid=0 2024-11-26T14:15:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741929_1105 (size=9857) 2024-11-26T14:15:22,178 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 2024-11-26T14:15:22,194 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c691d0007b8742f99a97c8e31576bf2c is 50, key is test_row_1/B:col10/1732630518835/Put/seqid=0 2024-11-26T14:15:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741930_1106 (size=9857) 2024-11-26T14:15:22,201 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c691d0007b8742f99a97c8e31576bf2c 2024-11-26T14:15:22,212 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1bd3bb2dc6b54cfda18ee32ac5b48023 is 50, key is test_row_1/C:col10/1732630518835/Put/seqid=0 2024-11-26T14:15:22,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741931_1107 (size=9857) 2024-11-26T14:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T14:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T14:15:22,622 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1bd3bb2dc6b54cfda18ee32ac5b48023 2024-11-26T14:15:22,628 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 2024-11-26T14:15:22,633 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/48e2a4b93dc64f47837e9b8ebf2ad1a0, entries=100, sequenceid=396, filesize=9.6 K 2024-11-26T14:15:22,634 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/B/c691d0007b8742f99a97c8e31576bf2c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c691d0007b8742f99a97c8e31576bf2c 2024-11-26T14:15:22,639 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c691d0007b8742f99a97c8e31576bf2c, entries=100, sequenceid=396, filesize=9.6 K 2024-11-26T14:15:22,640 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/.tmp/C/1bd3bb2dc6b54cfda18ee32ac5b48023 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1bd3bb2dc6b54cfda18ee32ac5b48023 2024-11-26T14:15:22,646 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1bd3bb2dc6b54cfda18ee32ac5b48023, entries=100, sequenceid=396, filesize=9.6 K 2024-11-26T14:15:22,647 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c77bbfdf456d364621817f40b0d9aed2 in 494ms, sequenceid=396, compaction requested=true 2024-11-26T14:15:22,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6872ae8ff4124a04ad32e19cb12223ef, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/ff00af8080764170b0c09399f52e7720, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d118a1ec9ebc408a9edfbea6e4d434a6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7e82e468acc64a13a57d3b75162420c7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/097edb0379f5444788402a58390826ed, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/da9df16d6fe945ab858f8002c1b9d2ac, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/20fd084d583f4dc0a8ff94a484e46a87, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0f9420157e2143aeacd338c8d63f6032, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425] to archive 2024-11-26T14:15:22,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:22,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/5900687dd65f4432be564bc96dc94688 2024-11-26T14:15:22,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0e24536015a14f0f8c4b98d639c9248e 2024-11-26T14:15:22,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/b3bb50645e56445e881d840d770a5c8f 2024-11-26T14:15:22,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6872ae8ff4124a04ad32e19cb12223ef to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6872ae8ff4124a04ad32e19cb12223ef 2024-11-26T14:15:22,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0039a9fff6f14a0dae51be20dcdf6e6e 2024-11-26T14:15:22,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/6ffed9d952744ffaa46b6be10355d783 2024-11-26T14:15:22,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/ff00af8080764170b0c09399f52e7720 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/ff00af8080764170b0c09399f52e7720 2024-11-26T14:15:22,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/a840a8e981c34699a5c9fa02e6da8d13 2024-11-26T14:15:22,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d118a1ec9ebc408a9edfbea6e4d434a6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d118a1ec9ebc408a9edfbea6e4d434a6 2024-11-26T14:15:22,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3df56bd335fd4fa49acec3d2a33e886c 2024-11-26T14:15:22,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/c5ea7e796fe942fd889f00d6df22ba4a 2024-11-26T14:15:22,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7e82e468acc64a13a57d3b75162420c7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7e82e468acc64a13a57d3b75162420c7 2024-11-26T14:15:22,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/1d4350e2b987409cb176d272229dbb7b 2024-11-26T14:15:22,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/d38a5f3caa1c4adca2553796ed005939 2024-11-26T14:15:22,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/26ebdd11fc67474eaccd1685402998f4 2024-11-26T14:15:22,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/097edb0379f5444788402a58390826ed to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/097edb0379f5444788402a58390826ed 2024-11-26T14:15:22,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/81c8071e147146cab8404fddf1ce407c 2024-11-26T14:15:22,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/924a1d2797574cfdb55db6b6c7520d9b 2024-11-26T14:15:22,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/7b9ed0b1dd824b27a285d58abd15623b 2024-11-26T14:15:22,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/da9df16d6fe945ab858f8002c1b9d2ac to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/da9df16d6fe945ab858f8002c1b9d2ac 2024-11-26T14:15:22,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/816cdc6587f7414f9bcfc646fac28001 2024-11-26T14:15:22,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/bca7b22e19c447f59e0559d7ddbd92b8 2024-11-26T14:15:22,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/20fd084d583f4dc0a8ff94a484e46a87 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/20fd084d583f4dc0a8ff94a484e46a87 2024-11-26T14:15:22,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/259455fe7f704463ae8cbc6e0bece095 2024-11-26T14:15:22,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/45aa222890ca4866aed75ccae920dfcd 2024-11-26T14:15:22,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f74dc525843d467e8352d0eddb03c5f7 2024-11-26T14:15:22,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0f9420157e2143aeacd338c8d63f6032 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/0f9420157e2143aeacd338c8d63f6032 2024-11-26T14:15:22,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/f5c8e2510868416bb4247aca99a33761 2024-11-26T14:15:22,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/46dff3df1d3943f7a82f542d158b7425 2024-11-26T14:15:22,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3f4f3666e20e45ce98c503b6418bc4dc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/ec2f5d3b3ba24802a7999a765155026d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3e804e5541bf421aad6d77d40bf203cf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/f5fa0346f91f438da5751eecabab9380, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/db1d44ebbbf14ea8a7688539274f024b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b685f674aaba43b7ba876c3cf81afb81, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/2e496b92c7cf480dacca14e61314fd49, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2] to archive 2024-11-26T14:15:22,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:22,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/d2d12f8c2c0f44198c697e7d952be226 2024-11-26T14:15:22,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/e21f4c0114b542afa5668b39fbc95b02 2024-11-26T14:15:22,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3f4f3666e20e45ce98c503b6418bc4dc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3f4f3666e20e45ce98c503b6418bc4dc 2024-11-26T14:15:22,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/cf092f8c946e4758bb347a29a7015400 2024-11-26T14:15:22,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9ebb16d3338d4865b197e17b06c1ad96 2024-11-26T14:15:22,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/ec2f5d3b3ba24802a7999a765155026d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/ec2f5d3b3ba24802a7999a765155026d 2024-11-26T14:15:22,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c90ea8e2153540c59e48e75a4cde5a44 2024-11-26T14:15:22,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3cb11089dc6b4f5b99501fc58a8b19ee 2024-11-26T14:15:22,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3e804e5541bf421aad6d77d40bf203cf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/3e804e5541bf421aad6d77d40bf203cf 2024-11-26T14:15:22,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b4782cb899ec4e8dbe2153588759e65a 2024-11-26T14:15:22,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/677baba90c354d34918009205eb14f05 2024-11-26T14:15:22,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/f5fa0346f91f438da5751eecabab9380 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/f5fa0346f91f438da5751eecabab9380 2024-11-26T14:15:22,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/fbc3c621cae74561a2bd255d1fbcb4e3 2024-11-26T14:15:22,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/583ffb2271ea4f44a3e952a3a49d6999 2024-11-26T14:15:22,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/db1d44ebbbf14ea8a7688539274f024b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/db1d44ebbbf14ea8a7688539274f024b 2024-11-26T14:15:22,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8e3f84a16bd64aaab00fdaa0716afd00 2024-11-26T14:15:22,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/9aa03e94c0e14a29b23d033996e573ea 2024-11-26T14:15:22,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/a3cd0faeffe341f5abc2dd9d68faf334 2024-11-26T14:15:22,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/88b21d0e13bf4f8ea7b6d14d98cfb9bf 2024-11-26T14:15:22,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/0322071bcf27484eac55a4fd5f25be07 2024-11-26T14:15:22,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/52a4f0dbe2b4428ebd23acd79ed617d5 2024-11-26T14:15:22,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b685f674aaba43b7ba876c3cf81afb81 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/b685f674aaba43b7ba876c3cf81afb81 2024-11-26T14:15:22,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/85bc205660d64aff9e2a87948bfaf163 2024-11-26T14:15:22,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/4ffa0d8e8b4649a8ae53d2e3e334db76 2024-11-26T14:15:22,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c018c3bc9bd04de5b3d4bc7274deca56 2024-11-26T14:15:22,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/2e496b92c7cf480dacca14e61314fd49 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/2e496b92c7cf480dacca14e61314fd49 2024-11-26T14:15:22,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/119160701a3141eb970fdb0d5f1ed180 2024-11-26T14:15:22,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/8487af05e1354750ab3f9ceecb656272 2024-11-26T14:15:22,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/93476a97d83743b78e64aed11e21ccb2 2024-11-26T14:15:22,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/798f1a5525794112858730da7b149b03, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/933cc75b51aa479e90cc5d12d9550ea7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/89f346745ef64fd0ac87cc98bf5f9b2c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1768c1ffb6864364a2e0658bf2ba633b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/b3a408803bea4986a477d4ba9f8c1480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/7bc4f8626a3f41bbabbaa2cb07c87c82, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/aed9ae38ebbd440e864058d5a207e905, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9] to archive 2024-11-26T14:15:22,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:22,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/ffb3d3c604ed416a8a8b7d9118b98602 2024-11-26T14:15:22,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/56249e48d62f4adaaafa5b0c71cca40e 2024-11-26T14:15:22,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/798f1a5525794112858730da7b149b03 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/798f1a5525794112858730da7b149b03 2024-11-26T14:15:22,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1f5441fbb1074ae3a30e9daacf3230f7 2024-11-26T14:15:22,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/9429e953358d4d1699bfeca4cf7f6b35 2024-11-26T14:15:22,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/933cc75b51aa479e90cc5d12d9550ea7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/933cc75b51aa479e90cc5d12d9550ea7 2024-11-26T14:15:22,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/44206b8e2fad49ffaa45349bc0fd403d 2024-11-26T14:15:22,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/24b0e508acd640c0af48e11a17945a0b 2024-11-26T14:15:22,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/89f346745ef64fd0ac87cc98bf5f9b2c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/89f346745ef64fd0ac87cc98bf5f9b2c 2024-11-26T14:15:22,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0319cf44ab86494ea26ed57cd101c0a7 2024-11-26T14:15:22,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/d5762320944241dcbf8d6b3edb1fa77b 2024-11-26T14:15:22,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1768c1ffb6864364a2e0658bf2ba633b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1768c1ffb6864364a2e0658bf2ba633b 2024-11-26T14:15:22,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/38df7549e3a846bfaa97f379bdfa8da4 2024-11-26T14:15:22,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/17214a2902b04d3f8ca5e97e7a15292c 2024-11-26T14:15:22,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/b3a408803bea4986a477d4ba9f8c1480 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/b3a408803bea4986a477d4ba9f8c1480 2024-11-26T14:15:22,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/18fbc8f28966415aa6a24a522a48df61 2024-11-26T14:15:22,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/5008dcb8a1d741529135191901d50090 2024-11-26T14:15:22,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6c593facc1741449bb97b23117e853e 2024-11-26T14:15:22,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/7bc4f8626a3f41bbabbaa2cb07c87c82 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/7bc4f8626a3f41bbabbaa2cb07c87c82 2024-11-26T14:15:22,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/0ce7d68dbb36475c815f080dc15f41ae 2024-11-26T14:15:22,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/732e1dd0ff7246b28bbe4e55433bf46c 2024-11-26T14:15:22,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/aed9ae38ebbd440e864058d5a207e905 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/aed9ae38ebbd440e864058d5a207e905 2024-11-26T14:15:22,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/c38799e3c8b349f6a799bfcb6383a09b 2024-11-26T14:15:22,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/e6c57d261dc94f98920c270905ccbdf7 2024-11-26T14:15:22,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/267fa12ff4ce486eb8b62bd0cc56cf22 2024-11-26T14:15:22,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/f6a61d7e198144b5b2eefd142e3a8d1c 2024-11-26T14:15:22,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a4042b172a344658aac00833d67f6af9 2024-11-26T14:15:22,867 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=1 2024-11-26T14:15:22,868 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. 2024-11-26T14:15:22,868 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for c77bbfdf456d364621817f40b0d9aed2: Waiting for close lock at 1732630522152Running coprocessor pre-close hooks at 1732630522152Disabling compacts and flushes for region at 1732630522152Disabling writes for close at 1732630522152Obtaining lock to block concurrent updates at 1732630522152Preparing flush snapshotting stores in c77bbfdf456d364621817f40b0d9aed2 at 1732630522152Finished memstore snapshotting TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732630522153 (+1 ms)Flushing stores of TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2. at 1732630522154 (+1 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/A: creating writer at 1732630522154Flushing c77bbfdf456d364621817f40b0d9aed2/A: appending metadata at 1732630522157 (+3 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/A: closing flushed file at 1732630522157Flushing c77bbfdf456d364621817f40b0d9aed2/B: creating writer at 1732630522188 (+31 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/B: appending metadata at 1732630522194 (+6 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/B: closing flushed file at 1732630522194Flushing c77bbfdf456d364621817f40b0d9aed2/C: creating writer at 1732630522208 (+14 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/C: appending metadata at 1732630522211 (+3 ms)Flushing c77bbfdf456d364621817f40b0d9aed2/C: closing flushed file at 1732630522211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7375d756: reopening flushed file at 1732630522627 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d65df6: reopening flushed file at 1732630522633 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@536800b6: reopening flushed file at 1732630522639 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c77bbfdf456d364621817f40b0d9aed2 in 494ms, sequenceid=396, compaction requested=true at 1732630522647 (+8 ms)Writing region close event to WAL at 1732630522862 (+215 ms)Running coprocessor post-close hooks at 1732630522867 (+5 ms)Closed at 1732630522867 2024-11-26T14:15:22,873 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:22,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=c77bbfdf456d364621817f40b0d9aed2, regionState=CLOSED 2024-11-26T14:15:22,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:22,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T14:15:22,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure c77bbfdf456d364621817f40b0d9aed2, server=19945b63c2b4,40025,1732630490995 in 888 msec 2024-11-26T14:15:22,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-26T14:15:22,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c77bbfdf456d364621817f40b0d9aed2, UNASSIGN in 906 msec 2024-11-26T14:15:22,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T14:15:22,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 918 msec 2024-11-26T14:15:22,894 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630522894"}]},"ts":"1732630522894"} 2024-11-26T14:15:22,896 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:15:22,896 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:15:22,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-26T14:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T14:15:23,103 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:23,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:15:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,112 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T14:15:23,115 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,121 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:23,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/recovered.edits] 2024-11-26T14:15:23,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3e81752d489b446ea42216f2d158b104 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/3e81752d489b446ea42216f2d158b104 2024-11-26T14:15:23,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/48e2a4b93dc64f47837e9b8ebf2ad1a0 2024-11-26T14:15:23,132 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/4a17daebb06a40b0a03d15efc8ed5a86 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/A/4a17daebb06a40b0a03d15efc8ed5a86 2024-11-26T14:15:23,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/46d72f4a99e642e89e32614de693c55c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/46d72f4a99e642e89e32614de693c55c 2024-11-26T14:15:23,136 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c691d0007b8742f99a97c8e31576bf2c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/c691d0007b8742f99a97c8e31576bf2c 2024-11-26T14:15:23,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/da3f448ef3c14ef6aa8d117a46355695 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/B/da3f448ef3c14ef6aa8d117a46355695 2024-11-26T14:15:23,140 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1bd3bb2dc6b54cfda18ee32ac5b48023 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/1bd3bb2dc6b54cfda18ee32ac5b48023 2024-11-26T14:15:23,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/3da22dd715df45968450cf0e4b402bd8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/3da22dd715df45968450cf0e4b402bd8 2024-11-26T14:15:23,144 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/52cb6e96f446444ca8932fa24d806dea to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/52cb6e96f446444ca8932fa24d806dea 2024-11-26T14:15:23,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a62107d405fb4b1db073466fa3281224 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/C/a62107d405fb4b1db073466fa3281224 2024-11-26T14:15:23,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/recovered.edits/399.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2/recovered.edits/399.seqid 2024-11-26T14:15:23,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/c77bbfdf456d364621817f40b0d9aed2 2024-11-26T14:15:23,149 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:15:23,154 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T14:15:23,165 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:15:23,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:15:23,209 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,209 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:15:23,210 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630523209"}]},"ts":"9223372036854775807"} 2024-11-26T14:15:23,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:15:23,223 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c77bbfdf456d364621817f40b0d9aed2, NAME => 'TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:15:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T14:15:23,223 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:15:23,223 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630523223"}]},"ts":"9223372036854775807"} 2024-11-26T14:15:23,236 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:15:23,237 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 130 msec 2024-11-26T14:15:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T14:15:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:15:23,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:23,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_289023253_22 at /127.0.0.1:37000 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;19945b63c2b4:40025-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=973 (was 795) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3898 (was 4498) 2024-11-26T14:15:23,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=973, ProcessCount=11, AvailableMemoryMB=3898 2024-11-26T14:15:23,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:15:23,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:15:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:23,472 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:15:23,472 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:23,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-26T14:15:23,473 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:15:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-26T14:15:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741932_1108 (size=1061) 2024-11-26T14:15:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-26T14:15:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-26T14:15:23,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:15:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741933_1109 (size=53) 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe0ec8e3e6af878b1c6bcf46977c8bf2, disabling compactions & flushes 2024-11-26T14:15:23,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. after waiting 0 ms 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:23,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:23,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: Waiting for close lock at 1732630523906Disabling compacts and flushes for region at 1732630523906Disabling writes for close at 1732630523906Writing region close event to WAL at 1732630523906Closed at 1732630523906 2024-11-26T14:15:23,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:15:23,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630523908"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630523908"}]},"ts":"1732630523908"} 2024-11-26T14:15:23,911 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:15:23,912 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:15:23,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630523912"}]},"ts":"1732630523912"} 2024-11-26T14:15:23,914 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:15:23,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, ASSIGN}] 2024-11-26T14:15:23,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, ASSIGN 2024-11-26T14:15:23,917 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:15:24,068 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:24,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, ASSIGN because future has completed 2024-11-26T14:15:24,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-26T14:15:24,232 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,233 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:15:24,233 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,233 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:15:24,234 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,234 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,235 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,237 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:24,238 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName A 2024-11-26T14:15:24,238 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:24,239 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:24,239 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,241 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:24,241 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName B 2024-11-26T14:15:24,241 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:24,242 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:24,242 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,243 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:24,243 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName C 2024-11-26T14:15:24,244 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:24,244 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:24,244 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,244 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,245 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,245 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,246 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,246 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,247 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:15:24,248 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,250 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:15:24,251 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened fe0ec8e3e6af878b1c6bcf46977c8bf2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69897243, jitterRate=0.041550084948539734}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:15:24,251 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,252 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: Running coprocessor pre-open hook at 1732630524234Writing region info on filesystem at 1732630524234Initializing all the Stores at 1732630524235 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630524235Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630524235Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630524235Cleaning up temporary data from old regions at 1732630524246 (+11 ms)Running coprocessor post-open hooks at 1732630524251 (+5 ms)Region opened successfully at 1732630524252 (+1 ms) 2024-11-26T14:15:24,252 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., pid=26, masterSystemTime=1732630524228 2024-11-26T14:15:24,255 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,255 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,255 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:24,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:24,272 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37705 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=19945b63c2b4,40025,1732630490995, table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T14:15:24,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T14:15:24,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 in 202 msec 2024-11-26T14:15:24,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-26T14:15:24,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, ASSIGN in 361 msec 2024-11-26T14:15:24,279 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:15:24,280 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630524279"}]},"ts":"1732630524279"} 2024-11-26T14:15:24,282 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:15:24,283 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:15:24,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 812 msec 2024-11-26T14:15:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-26T14:15:24,621 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:24,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:24,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:24,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:24,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f421ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:24,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236dd13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:24,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:24,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:24,650 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:15:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:15:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44cff70e 2024-11-26T14:15:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:15:24,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:15:24,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:15:24,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:15:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:24,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741934_1110 (size=1097) 2024-11-26T14:15:24,773 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T14:15:24,773 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T14:15:24,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:15:24,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE}] 2024-11-26T14:15:24,808 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE 2024-11-26T14:15:24,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:24,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE because future has completed 2024-11-26T14:15:24,833 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:15:24,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:24,993 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:24,993 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:15:24,993 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing fe0ec8e3e6af878b1c6bcf46977c8bf2, disabling compactions & flushes 2024-11-26T14:15:24,994 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,994 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:24,994 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. after waiting 0 ms 2024-11-26T14:15:24,994 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,024 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T14:15:25,026 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,026 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: Waiting for close lock at 1732630524993Running coprocessor pre-close hooks at 1732630524993Disabling compacts and flushes for region at 1732630524993Disabling writes for close at 1732630524994 (+1 ms)Writing region close event to WAL at 1732630524996 (+2 ms)Running coprocessor post-close hooks at 1732630525026 (+30 ms)Closed at 1732630525026 2024-11-26T14:15:25,026 WARN [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: fe0ec8e3e6af878b1c6bcf46977c8bf2 to self. 2024-11-26T14:15:25,038 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,039 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=CLOSED 2024-11-26T14:15:25,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:25,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T14:15:25,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 in 225 msec 2024-11-26T14:15:25,075 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE; state=CLOSED, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=true 2024-11-26T14:15:25,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:25,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE because future has completed 2024-11-26T14:15:25,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:25,398 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,399 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,404 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,405 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:25,411 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName A 2024-11-26T14:15:25,413 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:25,414 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:25,415 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,415 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:25,416 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName B 2024-11-26T14:15:25,416 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:25,417 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:25,417 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,418 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:25,418 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe0ec8e3e6af878b1c6bcf46977c8bf2 columnFamilyName C 2024-11-26T14:15:25,418 DEBUG [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:25,419 INFO [StoreOpener-fe0ec8e3e6af878b1c6bcf46977c8bf2-1 {}] regionserver.HStore(327): Store=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:25,419 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,419 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,420 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,421 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,422 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,422 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,423 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:15:25,424 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,425 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened fe0ec8e3e6af878b1c6bcf46977c8bf2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71065458, jitterRate=0.058957844972610474}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:15:25,425 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,427 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: Running coprocessor pre-open hook at 1732630525399Writing region info on filesystem at 1732630525399Initializing all the Stores at 1732630525400 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630525400Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630525401 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630525401Cleaning up temporary data from old regions at 1732630525422 (+21 ms)Running coprocessor post-open hooks at 1732630525425 (+3 ms)Region opened successfully at 1732630525426 (+1 ms) 2024-11-26T14:15:25,428 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., pid=31, masterSystemTime=1732630525395 2024-11-26T14:15:25,434 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,434 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,435 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=OPEN, openSeqNum=5, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:25,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:25,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-26T14:15:25,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 in 201 msec 2024-11-26T14:15:25,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-26T14:15:25,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, REOPEN/MOVE in 639 msec 2024-11-26T14:15:25,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T14:15:25,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 663 msec 2024-11-26T14:15:25,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 787 msec 2024-11-26T14:15:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T14:15:25,460 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:25,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc63b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb4dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b844252, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,493 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f2a62e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,508 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5397ea61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd63425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,531 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59622770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b58a086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aab77c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf4ad55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acf7b75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fa96c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37afb74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334c30fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,590 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4134c58b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,592 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2570f4e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:25,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:25,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e5b2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:25,600 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:25,603 DEBUG [Thread-572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,603 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,604 DEBUG [Thread-576 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,604 DEBUG [Thread-576 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,613 DEBUG [Thread-570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-26T14:15:25,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:25,620 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:25,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,626 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,633 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,638 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,641 DEBUG [Thread-583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,643 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,657 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,664 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:25,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:25,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:25,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:15:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:25,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:25,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e207db4f25bb4f87a50aa12ab4a76038_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630525646/Put/seqid=0 2024-11-26T14:15:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:25,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:25,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:25,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630525684, max timestamp=1732630525795Num uniques 859; 2024-11-26T14:15:25,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:25,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630525684, max timestamp=1732630525764Num uniques 859; 2024-11-26T14:15:25,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:25,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630525684, max timestamp=1732630525799Num uniques 859; 2024-11-26T14:15:25,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:26,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741935_1111 (size=17034) 2024-11-26T14:15:26,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630586064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:26,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630586073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e207db4f25bb4f87a50aa12ab4a76038_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e207db4f25bb4f87a50aa12ab4a76038_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630586074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630586081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/afb576ef3a6c45f2bb0a5946ac4b3a02, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:26,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/afb576ef3a6c45f2bb0a5946ac4b3a02 is 175, key is test_row_0/A:col10/1732630525646/Put/seqid=0 2024-11-26T14:15:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630586084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741936_1112 (size=48139) 2024-11-26T14:15:26,181 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/afb576ef3a6c45f2bb0a5946ac4b3a02 2024-11-26T14:15:26,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630586194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b7433e7efb74b39ae4d1a1d09d95b5e is 50, key is test_row_0/B:col10/1732630525646/Put/seqid=0 2024-11-26T14:15:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630586224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:26,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630586224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630586224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630586244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741937_1113 (size=12001) 2024-11-26T14:15:26,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b7433e7efb74b39ae4d1a1d09d95b5e 2024-11-26T14:15:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/b7d33dc4f26a4baf8b3b7ca492210ee9 is 50, key is test_row_0/C:col10/1732630525646/Put/seqid=0 2024-11-26T14:15:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630586413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741938_1114 (size=12001) 2024-11-26T14:15:26,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/b7d33dc4f26a4baf8b3b7ca492210ee9 2024-11-26T14:15:26,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/afb576ef3a6c45f2bb0a5946ac4b3a02 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02 2024-11-26T14:15:26,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02, entries=250, sequenceid=14, filesize=47.0 K 2024-11-26T14:15:26,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b7433e7efb74b39ae4d1a1d09d95b5e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e 2024-11-26T14:15:26,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T14:15:26,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/b7d33dc4f26a4baf8b3b7ca492210ee9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9 2024-11-26T14:15:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630586459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630586464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630586464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T14:15:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 835ms, sequenceid=14, compaction requested=false 2024-11-26T14:15:26,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:26,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630526065, max timestamp=1732630526503Num uniques 1009; 2024-11-26T14:15:26,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630526065, max timestamp=1732630526503Num uniques 1009; 2024-11-26T14:15:26,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:26,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=102.08 KB heapSize=290.38 KB 2024-11-26T14:15:26,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:26,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:26,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a91ce2e00c5c4dc897eeb0a11e2f90c3_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630526080/Put/seqid=0 2024-11-26T14:15:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741939_1115 (size=12154) 2024-11-26T14:15:26,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:26,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,639 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a91ce2e00c5c4dc897eeb0a11e2f90c3_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a91ce2e00c5c4dc897eeb0a11e2f90c3_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:26,641 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4723168d8ae0437894c53491d982f376, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:26,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4723168d8ae0437894c53491d982f376 is 175, key is test_row_0/A:col10/1732630526080/Put/seqid=0 2024-11-26T14:15:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741940_1116 (size=30955) 2024-11-26T14:15:26,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4723168d8ae0437894c53491d982f376 2024-11-26T14:15:26,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/10499d3217e541c0adbba06251a37373 is 50, key is test_row_0/B:col10/1732630526080/Put/seqid=0 2024-11-26T14:15:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:26,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630526732, max timestamp=1732630526771Num uniques 859; 2024-11-26T14:15:26,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:26,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630526732, max timestamp=1732630526771Num uniques 859; 2024-11-26T14:15:26,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:26,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741941_1117 (size=12001) 2024-11-26T14:15:26,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/10499d3217e541c0adbba06251a37373 2024-11-26T14:15:26,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630526732, max timestamp=1732630526771Num uniques 859; 2024-11-26T14:15:26,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:26,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa018d38cbb44cd99eec408ea1989d5a is 50, key is test_row_0/C:col10/1732630526080/Put/seqid=0 2024-11-26T14:15:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741942_1118 (size=12001) 2024-11-26T14:15:26,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa018d38cbb44cd99eec408ea1989d5a 2024-11-26T14:15:26,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4723168d8ae0437894c53491d982f376 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376 2024-11-26T14:15:26,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376, entries=150, sequenceid=52, filesize=30.2 K 2024-11-26T14:15:26,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/10499d3217e541c0adbba06251a37373 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373 2024-11-26T14:15:26,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T14:15:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa018d38cbb44cd99eec408ea1989d5a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a 2024-11-26T14:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630586906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630586907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630586908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630586911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630586917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:26,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:26,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:26,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-26T14:15:26,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=120.76 KB/123660 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 428ms, sequenceid=52, compaction requested=false 2024-11-26T14:15:26,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:27,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T14:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:27,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630587087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266cf5dd9703724c72997d5eb3172e6ec8_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630587102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630587104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630587128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741943_1119 (size=17034) 2024-11-26T14:15:27,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:27,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266cf5dd9703724c72997d5eb3172e6ec8_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266cf5dd9703724c72997d5eb3172e6ec8_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630587234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/8277290acd254ee99a9918c88d9836cc, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:27,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630587234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630587235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/8277290acd254ee99a9918c88d9836cc is 175, key is test_row_0/A:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630587249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630587284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741944_1120 (size=48139) 2024-11-26T14:15:27,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630587481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630587480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630587483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630587484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630587503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/8277290acd254ee99a9918c88d9836cc 2024-11-26T14:15:27,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/0605ce06fa5046d0aad4dbb06996c63a is 50, key is test_row_0/B:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:27,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741945_1121 (size=12001) 2024-11-26T14:15:27,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/0605ce06fa5046d0aad4dbb06996c63a 2024-11-26T14:15:27,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/7b69253d453245bb9dfd9f4e7864a901 is 50, key is test_row_0/C:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630587804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630587805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630587805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741946_1122 (size=12001) 2024-11-26T14:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630587817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630587814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:27,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/7b69253d453245bb9dfd9f4e7864a901 2024-11-26T14:15:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:27,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/8277290acd254ee99a9918c88d9836cc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc 2024-11-26T14:15:27,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc, entries=250, sequenceid=88, filesize=47.0 K 2024-11-26T14:15:27,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:27,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/0605ce06fa5046d0aad4dbb06996c63a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a 2024-11-26T14:15:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:28,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-26T14:15:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/7b69253d453245bb9dfd9f4e7864a901 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901 2024-11-26T14:15:28,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901, entries=150, sequenceid=88, filesize=11.7 K 2024-11-26T14:15:28,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 995ms, sequenceid=88, compaction requested=true 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:28,057 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:28,058 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:28,058 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:28,058 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,059 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=124.3 K 2024-11-26T14:15:28,059 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,059 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc] 2024-11-26T14:15:28,060 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:28,062 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb576ef3a6c45f2bb0a5946ac4b3a02, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732630525615 2024-11-26T14:15:28,067 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4723168d8ae0437894c53491d982f376, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732630526065 2024-11-26T14:15:28,068 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8277290acd254ee99a9918c88d9836cc, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526829 2024-11-26T14:15:28,084 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:28,084 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:28,084 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,084 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.2 K 2024-11-26T14:15:28,088 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b7433e7efb74b39ae4d1a1d09d95b5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732630525639 2024-11-26T14:15:28,090 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 10499d3217e541c0adbba06251a37373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732630526065 2024-11-26T14:15:28,095 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0605ce06fa5046d0aad4dbb06996c63a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526914 2024-11-26T14:15:28,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-26T14:15:28,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,123 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:28,125 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:28,136 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264da189c4ec534a618ce09d6e3078dde6_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:28,146 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264da189c4ec534a618ce09d6e3078dde6_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:28,146 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264da189c4ec534a618ce09d6e3078dde6_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:28,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126852b027235564db2880366e4ee4a44fe_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630527089/Put/seqid=0 2024-11-26T14:15:28,187 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:28,188 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/9586b51232aa4c8aab4645c67c4126f7 is 50, key is test_row_0/B:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741947_1123 (size=4469) 2024-11-26T14:15:28,199 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#104 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:28,200 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/001794d65a7b493c86507d52311f7e74 is 175, key is test_row_0/A:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:28,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741948_1124 (size=12154) 2024-11-26T14:15:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741949_1125 (size=12104) 2024-11-26T14:15:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741950_1126 (size=31058) 2024-11-26T14:15:28,330 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/9586b51232aa4c8aab4645c67c4126f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/9586b51232aa4c8aab4645c67c4126f7 2024-11-26T14:15:28,330 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/001794d65a7b493c86507d52311f7e74 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74 2024-11-26T14:15:28,346 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 001794d65a7b493c86507d52311f7e74(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:28,347 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:28,347 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630528057; duration=0sec 2024-11-26T14:15:28,348 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 9586b51232aa4c8aab4645c67c4126f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:28,348 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:28,349 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630528057; duration=0sec 2024-11-26T14:15:28,349 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:28,349 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:28,349 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:28,352 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:28,352 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:28,353 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:28,353 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:28,353 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,354 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.2 K 2024-11-26T14:15:28,355 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d33dc4f26a4baf8b3b7ca492210ee9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732630525639 2024-11-26T14:15:28,356 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa018d38cbb44cd99eec408ea1989d5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732630526065 2024-11-26T14:15:28,357 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b69253d453245bb9dfd9f4e7864a901, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526914 2024-11-26T14:15:28,379 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#107 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:28,379 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/82f63ebc469e481f8b6236e34e51fce4 is 50, key is test_row_0/C:col10/1732630527057/Put/seqid=0 2024-11-26T14:15:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630588405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630588406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630588406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630528385, max timestamp=1732630528406Num uniques 859; 2024-11-26T14:15:28,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:28,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630528385, max timestamp=1732630528406Num uniques 859; 2024-11-26T14:15:28,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:28,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630528385, max timestamp=1732630528406Num uniques 859; 2024-11-26T14:15:28,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:28,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741951_1127 (size=12104) 2024-11-26T14:15:28,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/82f63ebc469e481f8b6236e34e51fce4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/82f63ebc469e481f8b6236e34e51fce4 2024-11-26T14:15:28,460 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:15:28,499 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 82f63ebc469e481f8b6236e34e51fce4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:28,499 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:28,499 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630528057; duration=0sec 2024-11-26T14:15:28,500 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:28,500 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630588545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630588547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630588547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630588550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630588554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:28,640 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126852b027235564db2880366e4ee4a44fe_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126852b027235564db2880366e4ee4a44fe_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:28,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/18e018d36bc04947b1fc29059ca458f6, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:28,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/18e018d36bc04947b1fc29059ca458f6 is 175, key is test_row_0/A:col10/1732630527089/Put/seqid=0 2024-11-26T14:15:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630588676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741952_1128 (size=30955) 2024-11-26T14:15:28,684 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/18e018d36bc04947b1fc29059ca458f6 2024-11-26T14:15:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630588685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630588685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630588692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630588692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f76bfa76c06a4ba5832bd7ec36f1d14d is 50, key is test_row_0/B:col10/1732630527089/Put/seqid=0 2024-11-26T14:15:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741953_1129 (size=12001) 2024-11-26T14:15:28,747 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f76bfa76c06a4ba5832bd7ec36f1d14d 2024-11-26T14:15:28,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/794075aa655b44c8a0809b7f6d1a5c5c is 50, key is test_row_0/C:col10/1732630527089/Put/seqid=0 2024-11-26T14:15:28,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741954_1130 (size=12001) 2024-11-26T14:15:28,790 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/794075aa655b44c8a0809b7f6d1a5c5c 2024-11-26T14:15:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/18e018d36bc04947b1fc29059ca458f6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6 2024-11-26T14:15:28,834 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6, entries=150, sequenceid=99, filesize=30.2 K 2024-11-26T14:15:28,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f76bfa76c06a4ba5832bd7ec36f1d14d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d 2024-11-26T14:15:28,852 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T14:15:28,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/794075aa655b44c8a0809b7f6d1a5c5c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c 2024-11-26T14:15:28,875 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T14:15:28,877 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 754ms, sequenceid=99, compaction requested=false 2024-11-26T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-26T14:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-26T14:15:28,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-26T14:15:28,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2450 sec 2024-11-26T14:15:28,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 3.2670 sec 2024-11-26T14:15:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:28,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:28,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:28,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630588930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e2720ae9200744dfa360b1a633ffbde8_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630588921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630588935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630588940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630588948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741955_1131 (size=14794) 2024-11-26T14:15:28,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:29,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e2720ae9200744dfa360b1a633ffbde8_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2720ae9200744dfa360b1a633ffbde8_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:29,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/68131853465d43b288e414757c53a205, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:29,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/68131853465d43b288e414757c53a205 is 175, key is test_row_0/A:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630589043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741956_1132 (size=39749) 2024-11-26T14:15:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630589054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630589055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630589063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630589063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630589253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630589263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630589264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630589273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630589273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/68131853465d43b288e414757c53a205 2024-11-26T14:15:29,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f77fa7b3b67844fabb9950706af1838a is 50, key is test_row_0/B:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:29,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741957_1133 (size=12151) 2024-11-26T14:15:29,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f77fa7b3b67844fabb9950706af1838a 2024-11-26T14:15:29,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/de4a26dc9ee945218254a0e0b734efc2 is 50, key is test_row_0/C:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630589564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741958_1134 (size=12151) 2024-11-26T14:15:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630589583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630589583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630589588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630589593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T14:15:29,797 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:29,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-26T14:15:29,822 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:29,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:29,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:29,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/de4a26dc9ee945218254a0e0b734efc2 2024-11-26T14:15:29,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T14:15:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:29,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:30,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/68131853465d43b288e414757c53a205 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205 2024-11-26T14:15:30,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205, entries=200, sequenceid=139, filesize=38.8 K 2024-11-26T14:15:30,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/f77fa7b3b67844fabb9950706af1838a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a 2024-11-26T14:15:30,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a, entries=150, sequenceid=139, filesize=11.9 K 2024-11-26T14:15:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/de4a26dc9ee945218254a0e0b734efc2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2 2024-11-26T14:15:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630590086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:30,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630590105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:30,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630590105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:30,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630590116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:30,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630590116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:30,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2, entries=150, sequenceid=139, filesize=11.9 K 2024-11-26T14:15:30,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1228ms, sequenceid=139, compaction requested=true 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:30,137 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:30,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:30,138 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:30,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-26T14:15:30,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,147 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:30,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:30,147 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:30,148 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:30,148 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,148 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=99.4 K 2024-11-26T14:15:30,148 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,148 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205] 2024-11-26T14:15:30,149 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:30,149 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:30,149 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,149 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/9586b51232aa4c8aab4645c67c4126f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.4 K 2024-11-26T14:15:30,152 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9586b51232aa4c8aab4645c67c4126f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526914 2024-11-26T14:15:30,152 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001794d65a7b493c86507d52311f7e74, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526914 2024-11-26T14:15:30,157 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f76bfa76c06a4ba5832bd7ec36f1d14d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630527089 2024-11-26T14:15:30,157 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e018d36bc04947b1fc29059ca458f6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630527089 2024-11-26T14:15:30,158 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68131853465d43b288e414757c53a205, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:30,160 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f77fa7b3b67844fabb9950706af1838a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:30,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264cf83af3704941f78edc6b2de5a24ed0_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630528912/Put/seqid=0 2024-11-26T14:15:30,197 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:30,210 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d0c22ce6fb054ef395b00d3cfbaded6a_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:30,217 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#115 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:30,219 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d0c22ce6fb054ef395b00d3cfbaded6a_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:30,219 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d0c22ce6fb054ef395b00d3cfbaded6a_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:30,220 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/4e44a73ea89a4f87ab557cd2fe6ca7db is 50, key is test_row_0/B:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741959_1135 (size=12304) 2024-11-26T14:15:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,254 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264cf83af3704941f78edc6b2de5a24ed0_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cf83af3704941f78edc6b2de5a24ed0_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/2ecd4715b2ab42268a569c4c48f28eab, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/2ecd4715b2ab42268a569c4c48f28eab is 175, key is test_row_0/A:col10/1732630528912/Put/seqid=0 2024-11-26T14:15:30,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741961_1137 (size=12357) 2024-11-26T14:15:30,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741960_1136 (size=4469) 2024-11-26T14:15:30,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741962_1138 (size=31105) 2024-11-26T14:15:30,340 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/2ecd4715b2ab42268a569c4c48f28eab 2024-11-26T14:15:30,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5b701cf3a3494bd1af1cc17023c42161 is 50, key is test_row_0/B:col10/1732630528912/Put/seqid=0 2024-11-26T14:15:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741963_1139 (size=12151) 2024-11-26T14:15:30,446 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5b701cf3a3494bd1af1cc17023c42161 2024-11-26T14:15:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/849fb12528db444da564fbf709f557dc is 50, key is test_row_0/C:col10/1732630528912/Put/seqid=0 2024-11-26T14:15:30,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741964_1140 (size=12151) 2024-11-26T14:15:30,716 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#114 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:30,717 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e04babee93154eb4ae3366fcc701a02a is 175, key is test_row_0/A:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:30,756 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/4e44a73ea89a4f87ab557cd2fe6ca7db as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4e44a73ea89a4f87ab557cd2fe6ca7db 2024-11-26T14:15:30,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741965_1141 (size=31311) 2024-11-26T14:15:30,778 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 4e44a73ea89a4f87ab557cd2fe6ca7db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:30,779 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:30,779 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630530137; duration=0sec 2024-11-26T14:15:30,779 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:30,779 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:30,779 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:30,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:30,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:30,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/82f63ebc469e481f8b6236e34e51fce4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.4 K 2024-11-26T14:15:30,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f63ebc469e481f8b6236e34e51fce4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732630526914 2024-11-26T14:15:30,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 794075aa655b44c8a0809b7f6d1a5c5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630527089 2024-11-26T14:15:30,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting de4a26dc9ee945218254a0e0b734efc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:30,798 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:30,799 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/28227833f475455189b0d2c95e38dff1 is 50, key is test_row_0/C:col10/1732630528899/Put/seqid=0 2024-11-26T14:15:30,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741966_1142 (size=12357) 2024-11-26T14:15:30,932 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/849fb12528db444da564fbf709f557dc 2024-11-26T14:15:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/2ecd4715b2ab42268a569c4c48f28eab as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab 2024-11-26T14:15:30,951 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab, entries=150, sequenceid=148, filesize=30.4 K 2024-11-26T14:15:30,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5b701cf3a3494bd1af1cc17023c42161 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161 2024-11-26T14:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,962 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T14:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:30,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/849fb12528db444da564fbf709f557dc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc 2024-11-26T14:15:30,974 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T14:15:30,975 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 829ms, sequenceid=148, compaction requested=false 2024-11-26T14:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-26T14:15:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-26T14:15:30,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T14:15:30,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-11-26T14:15:30,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.1840 sec 2024-11-26T14:15:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,212 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e04babee93154eb4ae3366fcc701a02a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a 2024-11-26T14:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,244 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into e04babee93154eb4ae3366fcc701a02a(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-26T14:15:31,244 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:31,244 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630530137; duration=1sec 2024-11-26T14:15:31,244 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:31,244 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262555395c54604a8c9f41cf5ca9d2c7d4_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,276 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/28227833f475455189b0d2c95e38dff1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/28227833f475455189b0d2c95e38dff1 2024-11-26T14:15:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,283 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 28227833f475455189b0d2c95e38dff1(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:31,283 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,284 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630530137; duration=0sec 2024-11-26T14:15:31,284 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:31,284 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741967_1143 (size=22268) 2024-11-26T14:15:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630531303, max timestamp=1732630531346Num uniques 859; 2024-11-26T14:15:31,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:31,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630531303, max timestamp=1732630531346Num uniques 859; 2024-11-26T14:15:31,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:31,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630531303, max timestamp=1732630531346Num uniques 859; 2024-11-26T14:15:31,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630591393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630591395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630591404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630591408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630591409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630591513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630591523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630591523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630591534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630591534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:31,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262555395c54604a8c9f41cf5ca9d2c7d4_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262555395c54604a8c9f41cf5ca9d2c7d4_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:31,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5b34f236de54429f8d6870b5f25dc89b, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:31,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5b34f236de54429f8d6870b5f25dc89b is 175, key is test_row_0/A:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630591725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630591734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630591734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741968_1144 (size=65673) 2024-11-26T14:15:31,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5b34f236de54429f8d6870b5f25dc89b 2024-11-26T14:15:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630591760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630591760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:31,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fed66327fd594b579097b329b9b0da0e is 50, key is test_row_0/B:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:31,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741969_1145 (size=12151) 2024-11-26T14:15:31,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fed66327fd594b579097b329b9b0da0e 2024-11-26T14:15:31,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/e0ff1669dd5e485a92485f4a74d793be is 50, key is test_row_0/C:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741970_1146 (size=12151) 2024-11-26T14:15:31,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/e0ff1669dd5e485a92485f4a74d793be 2024-11-26T14:15:31,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5b34f236de54429f8d6870b5f25dc89b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b 2024-11-26T14:15:31,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b, entries=350, sequenceid=159, filesize=64.1 K 2024-11-26T14:15:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T14:15:31,984 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:31,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-26T14:15:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:31,989 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:31,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fed66327fd594b579097b329b9b0da0e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e 2024-11-26T14:15:31,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:31,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:32,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T14:15:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/e0ff1669dd5e485a92485f4a74d793be as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be 2024-11-26T14:15:32,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T14:15:32,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 799ms, sequenceid=159, compaction requested=true 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:32,026 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:32,027 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:32,027 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:32,027 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,027 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=125.1 K 2024-11-26T14:15:32,028 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,028 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b] 2024-11-26T14:15:32,028 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e04babee93154eb4ae3366fcc701a02a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:32,029 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ecd4715b2ab42268a569c4c48f28eab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630528912 2024-11-26T14:15:32,029 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b34f236de54429f8d6870b5f25dc89b, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531187 2024-11-26T14:15:32,032 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:32,043 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:32,043 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:32,043 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,043 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4e44a73ea89a4f87ab557cd2fe6ca7db, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.8 K 2024-11-26T14:15:32,048 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e44a73ea89a4f87ab557cd2fe6ca7db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:32,052 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b701cf3a3494bd1af1cc17023c42161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630528912 2024-11-26T14:15:32,056 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed66327fd594b579097b329b9b0da0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531205 2024-11-26T14:15:32,062 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:32,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T14:15:32,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:32,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:32,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:32,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:32,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:32,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:32,073 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411269e99c4dbc25d448dba65a79a378001b5_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:32,076 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411269e99c4dbc25d448dba65a79a378001b5_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:32,077 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269e99c4dbc25d448dba65a79a378001b5_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630592086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630592090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630592093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630592097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262c0823de3cf14662bdd81d8c6e653a8c_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630531405/Put/seqid=0 2024-11-26T14:15:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630592101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,149 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:32,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/4906bb66c1534e879131960f032437cd is 50, key is test_row_0/B:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:32,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741972_1148 (size=14794) 2024-11-26T14:15:32,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262c0823de3cf14662bdd81d8c6e653a8c_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c0823de3cf14662bdd81d8c6e653a8c_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:32,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5663a29b684944e1b2af80ff642b0488, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:32,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5663a29b684944e1b2af80ff642b0488 is 175, key is test_row_0/A:col10/1732630531405/Put/seqid=0 2024-11-26T14:15:32,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741971_1147 (size=4469) 2024-11-26T14:15:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741973_1149 (size=12459) 2024-11-26T14:15:32,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741974_1150 (size=39749) 2024-11-26T14:15:32,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630592213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5663a29b684944e1b2af80ff642b0488 2024-11-26T14:15:32,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630592213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630592213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,230 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/4906bb66c1534e879131960f032437cd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4906bb66c1534e879131960f032437cd 2024-11-26T14:15:32,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630592224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fcaf00fc316745fe8e506ad8dea92831 is 50, key is test_row_0/B:col10/1732630531405/Put/seqid=0 2024-11-26T14:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,255 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 4906bb66c1534e879131960f032437cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:32,255 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:32,255 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630532026; duration=0sec 2024-11-26T14:15:32,255 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:32,255 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:32,255 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:32,263 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:32,263 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:32,263 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,264 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/28227833f475455189b0d2c95e38dff1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=35.8 K 2024-11-26T14:15:32,264 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28227833f475455189b0d2c95e38dff1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732630528522 2024-11-26T14:15:32,265 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849fb12528db444da564fbf709f557dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630528912 2024-11-26T14:15:32,265 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ff1669dd5e485a92485f4a74d793be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531205 2024-11-26T14:15:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741975_1151 (size=12151) 2024-11-26T14:15:32,293 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:32,293 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/178f5daa65da42e4bfb5c200162cd485 is 50, key is test_row_0/C:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:32,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fcaf00fc316745fe8e506ad8dea92831 2024-11-26T14:15:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:32,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741976_1152 (size=12459) 2024-11-26T14:15:32,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/a9a678b4448244df89f2376ac19ff93d is 50, key is test_row_0/C:col10/1732630531405/Put/seqid=0 2024-11-26T14:15:32,395 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/178f5daa65da42e4bfb5c200162cd485 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/178f5daa65da42e4bfb5c200162cd485 2024-11-26T14:15:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741977_1153 (size=12151) 2024-11-26T14:15:32,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/a9a678b4448244df89f2376ac19ff93d 2024-11-26T14:15:32,422 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 178f5daa65da42e4bfb5c200162cd485(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:32,422 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:32,422 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630532026; duration=0sec 2024-11-26T14:15:32,422 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:32,422 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630592424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630592424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/5663a29b684944e1b2af80ff642b0488 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488 2024-11-26T14:15:32,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630592433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630592455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488, entries=200, sequenceid=198, filesize=38.8 K 2024-11-26T14:15:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fcaf00fc316745fe8e506ad8dea92831 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831 2024-11-26T14:15:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T14:15:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/a9a678b4448244df89f2376ac19ff93d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d 2024-11-26T14:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T14:15:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 459ms, sequenceid=198, compaction requested=false 2024-11-26T14:15:32,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,592 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#122 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:32,593 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a1f7499fbebd40b6ae3e3ef53d01babf is 175, key is test_row_0/A:col10/1732630531205/Put/seqid=0 2024-11-26T14:15:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741978_1154 (size=31413) 2024-11-26T14:15:32,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:32,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:32,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,634 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a1f7499fbebd40b6ae3e3ef53d01babf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf 2024-11-26T14:15:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261f2a023edbc14b5dba024ffd8ed678e3_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,655 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into a1f7499fbebd40b6ae3e3ef53d01babf(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,655 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:32,655 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630532026; duration=0sec 2024-11-26T14:15:32,655 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:32,655 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741979_1155 (size=12304) 2024-11-26T14:15:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630592775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630592784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630592791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630532767, max timestamp=1732630532786Num uniques 859; 2024-11-26T14:15:32,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:32,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630532745, max timestamp=1732630532786Num uniques 859; 2024-11-26T14:15:32,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:32,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630532745, max timestamp=1732630532786Num uniques 859; 2024-11-26T14:15:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630592882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630592896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630592904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630592904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630592904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:32,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:32,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630593040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630593041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:33,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261f2a023edbc14b5dba024ffd8ed678e3_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261f2a023edbc14b5dba024ffd8ed678e3_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:33,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e232c00900174aef8f8c3895a8cfeaa8, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e232c00900174aef8f8c3895a8cfeaa8 is 175, key is test_row_0/A:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:33,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741980_1156 (size=31101) 2024-11-26T14:15:33,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e232c00900174aef8f8c3895a8cfeaa8 2024-11-26T14:15:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630593126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630593128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630593128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:33,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/16f6e55ae63a45f5b8d416b47ac5ad93 is 50, key is test_row_0/B:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741981_1157 (size=9757) 2024-11-26T14:15:33,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/16f6e55ae63a45f5b8d416b47ac5ad93 2024-11-26T14:15:33,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa55db99a82e4b9c9e51d0018c50f15c is 50, key is test_row_0/C:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630593260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630593273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741982_1158 (size=9757) 2024-11-26T14:15:33,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa55db99a82e4b9c9e51d0018c50f15c 2024-11-26T14:15:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/e232c00900174aef8f8c3895a8cfeaa8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8 2024-11-26T14:15:33,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:33,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:33,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:33,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8, entries=150, sequenceid=210, filesize=30.4 K 2024-11-26T14:15:33,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/16f6e55ae63a45f5b8d416b47ac5ad93 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93 2024-11-26T14:15:33,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93, entries=100, sequenceid=210, filesize=9.5 K 2024-11-26T14:15:33,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/aa55db99a82e4b9c9e51d0018c50f15c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c 2024-11-26T14:15:33,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c, entries=100, sequenceid=210, filesize=9.5 K 2024-11-26T14:15:33,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.00 KB/136187 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 717ms, sequenceid=210, compaction requested=true 2024-11-26T14:15:33,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:33,341 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:33,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:33,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:33,342 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:33,345 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:33,345 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:33,346 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,346 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4906bb66c1534e879131960f032437cd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=33.6 K 2024-11-26T14:15:33,345 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:33,347 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:33,347 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,348 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=99.9 K 2024-11-26T14:15:33,348 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,348 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8] 2024-11-26T14:15:33,348 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4906bb66c1534e879131960f032437cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531205 2024-11-26T14:15:33,348 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting fcaf00fc316745fe8e506ad8dea92831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732630531395 2024-11-26T14:15:33,349 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f7499fbebd40b6ae3e3ef53d01babf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531205 2024-11-26T14:15:33,349 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f6e55ae63a45f5b8d416b47ac5ad93, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630532096 2024-11-26T14:15:33,349 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5663a29b684944e1b2af80ff642b0488, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732630531386 2024-11-26T14:15:33,350 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e232c00900174aef8f8c3895a8cfeaa8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630532088 2024-11-26T14:15:33,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:33,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:33,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:33,362 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:33,363 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fe044ad57cbe4be1b3a322e4e4d92661 is 50, key is test_row_0/B:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,365 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,401 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411260a678eb19f46499d8a878e344b0947b5_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741983_1159 (size=12561) 2024-11-26T14:15:33,403 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411260a678eb19f46499d8a878e344b0947b5_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,403 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260a678eb19f46499d8a878e344b0947b5_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,416 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/fe044ad57cbe4be1b3a322e4e4d92661 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fe044ad57cbe4be1b3a322e4e4d92661 2024-11-26T14:15:33,422 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into fe044ad57cbe4be1b3a322e4e4d92661(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:33,422 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:33,422 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630533342; duration=0sec 2024-11-26T14:15:33,424 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:33,424 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741984_1160 (size=4469) 2024-11-26T14:15:33,427 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:33,437 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#132 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:33,438 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c1bbcb32915f41dab2cd36c9c35c7480 is 175, key is test_row_0/A:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,439 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:33,439 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:33,439 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,439 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/178f5daa65da42e4bfb5c200162cd485, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=33.6 K 2024-11-26T14:15:33,444 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 178f5daa65da42e4bfb5c200162cd485, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732630531205 2024-11-26T14:15:33,445 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a678b4448244df89f2376ac19ff93d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732630531395 2024-11-26T14:15:33,449 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa55db99a82e4b9c9e51d0018c50f15c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630532096 2024-11-26T14:15:33,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630532800, max timestamp=1732630533448Num uniques 1009; 2024-11-26T14:15:33,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:33,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T14:15:33,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:33,455 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=102.08 KB heapSize=290.38 KB 2024-11-26T14:15:33,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630532800, max timestamp=1732630533448Num uniques 1009; 2024-11-26T14:15:33,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630532800, max timestamp=1732630533448Num uniques 1009; 2024-11-26T14:15:33,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:33,469 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:33,471 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/9690614e2b174b38898d8efe515fbb27 is 50, key is test_row_0/C:col10/1732630532621/Put/seqid=0 2024-11-26T14:15:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:33,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112672c9aea619bc4bc29ee9b39e63d8876a_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630533448/Put/seqid=0 2024-11-26T14:15:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741985_1161 (size=31622) 2024-11-26T14:15:33,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741986_1162 (size=12561) 2024-11-26T14:15:33,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741987_1163 (size=17284) 2024-11-26T14:15:33,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:33,543 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/9690614e2b174b38898d8efe515fbb27 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9690614e2b174b38898d8efe515fbb27 2024-11-26T14:15:33,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630533494, max timestamp=1732630533535Num uniques 859; 2024-11-26T14:15:33,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630533494, max timestamp=1732630533535Num uniques 859; 2024-11-26T14:15:33,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:33,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:33,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630533494, max timestamp=1732630533535Num uniques 859; 2024-11-26T14:15:33,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:33,544 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c1bbcb32915f41dab2cd36c9c35c7480 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480 2024-11-26T14:15:33,550 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112672c9aea619bc4bc29ee9b39e63d8876a_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112672c9aea619bc4bc29ee9b39e63d8876a_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a64861e1adaa48b490075988c3749346, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:33,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a64861e1adaa48b490075988c3749346 is 175, key is test_row_0/A:col10/1732630533448/Put/seqid=0 2024-11-26T14:15:33,570 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 9690614e2b174b38898d8efe515fbb27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:33,570 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:33,570 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630533351; duration=0sec 2024-11-26T14:15:33,570 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:33,570 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:33,579 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into c1bbcb32915f41dab2cd36c9c35c7480(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:33,579 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:33,579 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630533341; duration=0sec 2024-11-26T14:15:33,579 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:33,579 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741988_1164 (size=48389) 2024-11-26T14:15:33,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630533599, max timestamp=1732630533641Num uniques 1068; 2024-11-26T14:15:33,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630533599, max timestamp=1732630533641Num uniques 1068; 2024-11-26T14:15:33,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T14:15:33,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T14:15:33,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630533599, max timestamp=1732630533641Num uniques 1068; 2024-11-26T14:15:33,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T14:15:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630533694, max timestamp=1732630533734Num uniques 1077; 2024-11-26T14:15:33,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T14:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630533694, max timestamp=1732630533734Num uniques 1077; 2024-11-26T14:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T14:15:33,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630533694, max timestamp=1732630533734Num uniques 1077; 2024-11-26T14:15:33,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T14:15:33,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630533826, max timestamp=1732630533857Num uniques 1086; 2024-11-26T14:15:33,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:15:33,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630533826, max timestamp=1732630533857Num uniques 1086; 2024-11-26T14:15:33,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:15:33,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630533826, max timestamp=1732630533857Num uniques 1086; 2024-11-26T14:15:33,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:15:33,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630593910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630593921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630533901, max timestamp=1732630533919Num uniques 1045; 2024-11-26T14:15:33,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-26T14:15:33,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630533901, max timestamp=1732630533919Num uniques 1045; 2024-11-26T14:15:33,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-26T14:15:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:33,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630533901, max timestamp=1732630533919Num uniques 1045; 2024-11-26T14:15:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-26T14:15:33,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630533962, max timestamp=1732630533983Num uniques 1009; 2024-11-26T14:15:33,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-26T14:15:33,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630533962, max timestamp=1732630533986Num uniques 1009; 2024-11-26T14:15:33,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-26T14:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630533962, max timestamp=1732630533983Num uniques 1009; 2024-11-26T14:15:33,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-26T14:15:34,033 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a64861e1adaa48b490075988c3749346 2024-11-26T14:15:34,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/469a032d8a5045d59880d55c265a4e45 is 50, key is test_row_0/B:col10/1732630533448/Put/seqid=0 2024-11-26T14:15:34,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732630534023, max timestamp=1732630534048Num uniques 1063; 2024-11-26T14:15:34,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=16618, suffix heap size=138520,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=363 2024-11-26T14:15:34,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732630534023, max timestamp=1732630534048Num uniques 1063; 2024-11-26T14:15:34,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=16618, suffix heap size=138520,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=363 2024-11-26T14:15:34,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732630534023, max timestamp=1732630534048Num uniques 1063; 2024-11-26T14:15:34,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=16618, suffix heap size=138520,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=363 2024-11-26T14:15:34,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741989_1165 (size=12151) 2024-11-26T14:15:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630594141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630594143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630594144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630594145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630594149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630594253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630594268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630594269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630594268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630594269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630594475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630594475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630594475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630594485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630594485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,496 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/469a032d8a5045d59880d55c265a4e45 2024-11-26T14:15:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/84ea066104b64849acb933838d7a2d84 is 50, key is test_row_0/C:col10/1732630533448/Put/seqid=0 2024-11-26T14:15:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741990_1166 (size=12151) 2024-11-26T14:15:34,548 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/84ea066104b64849acb933838d7a2d84 2024-11-26T14:15:34,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a64861e1adaa48b490075988c3749346 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346 2024-11-26T14:15:34,569 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346, entries=250, sequenceid=251, filesize=47.3 K 2024-11-26T14:15:34,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/469a032d8a5045d59880d55c265a4e45 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45 2024-11-26T14:15:34,597 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45, entries=150, sequenceid=251, filesize=11.9 K 2024-11-26T14:15:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/84ea066104b64849acb933838d7a2d84 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84 2024-11-26T14:15:34,647 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84, entries=150, sequenceid=251, filesize=11.9 K 2024-11-26T14:15:34,649 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=161.02 KB/164880 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1194ms, sequenceid=251, compaction requested=false 2024-11-26T14:15:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-26T14:15:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-26T14:15:34,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-26T14:15:34,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6560 sec 2024-11-26T14:15:34,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.6770 sec 2024-11-26T14:15:34,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=181.14 KB heapSize=506.02 KB 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:34,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630594803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630594833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630594834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630594835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630594842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264cacf4c2904c43b4803b347248a72fbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:34,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741991_1167 (size=12454) 2024-11-26T14:15:34,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:34,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264cacf4c2904c43b4803b347248a72fbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cacf4c2904c43b4803b347248a72fbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:34,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/20007f61623f4d5fb7b41b6e1bcaab43, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:34,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/20007f61623f4d5fb7b41b6e1bcaab43 is 175, key is test_row_0/A:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:34,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741992_1168 (size=31255) 2024-11-26T14:15:34,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/20007f61623f4d5fb7b41b6e1bcaab43 2024-11-26T14:15:34,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630594967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630594968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/378d35eda33b4d2689aadbdc5d9db5b8 is 50, key is test_row_0/B:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630594968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630594969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741993_1169 (size=12301) 2024-11-26T14:15:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630595183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630595183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630595184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630595193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630595353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/378d35eda33b4d2689aadbdc5d9db5b8 2024-11-26T14:15:35,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5658de34b0024718a5c32784e6fc7809 is 50, key is test_row_0/C:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:35,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741994_1170 (size=12301) 2024-11-26T14:15:35,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5658de34b0024718a5c32784e6fc7809 2024-11-26T14:15:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630595493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/20007f61623f4d5fb7b41b6e1bcaab43 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43 2024-11-26T14:15:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630595504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:35,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43, entries=150, sequenceid=396, filesize=30.5 K 2024-11-26T14:15:35,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/378d35eda33b4d2689aadbdc5d9db5b8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8 2024-11-26T14:15:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T14:15:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5658de34b0024718a5c32784e6fc7809 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809 2024-11-26T14:15:35,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809, entries=150, sequenceid=396, filesize=12.0 K 2024-11-26T14:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~505.97 KB/518112, currentSize=6.71 KB/6870 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 716ms, sequenceid=396, compaction requested=true 2024-11-26T14:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:35,550 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:35,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,552 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,555 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111266 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:35,555 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:35,556 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:35,556 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=108.7 K 2024-11-26T14:15:35,556 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:35,556 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43] 2024-11-26T14:15:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,558 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c1bbcb32915f41dab2cd36c9c35c7480, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630531395 2024-11-26T14:15:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,558 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a64861e1adaa48b490075988c3749346, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732630532800 2024-11-26T14:15:35,559 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 20007f61623f4d5fb7b41b6e1bcaab43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,565 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:35,565 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:35,565 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:35,565 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fe044ad57cbe4be1b3a322e4e4d92661, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=36.1 K 2024-11-26T14:15:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,570 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe044ad57cbe4be1b3a322e4e4d92661, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630531395 2024-11-26T14:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,573 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469a032d8a5045d59880d55c265a4e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732630532864 2024-11-26T14:15:35,573 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 378d35eda33b4d2689aadbdc5d9db5b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,575 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,585 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126681f1c93ee644d73b442d61d6fe90ec5_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,587 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126681f1c93ee644d73b442d61d6fe90ec5_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:35,587 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126681f1c93ee644d73b442d61d6fe90ec5_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741995_1171 (size=4469) 2024-11-26T14:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,633 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,634 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/1ecd6c19c4114db98b7c5f2bca9a556a is 50, key is test_row_0/B:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741996_1172 (size=12813) 2024-11-26T14:15:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,681 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/1ecd6c19c4114db98b7c5f2bca9a556a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1ecd6c19c4114db98b7c5f2bca9a556a 2024-11-26T14:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,693 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 1ecd6c19c4114db98b7c5f2bca9a556a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:35,693 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:35,693 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630535551; duration=0sec 2024-11-26T14:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,693 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:35,694 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,694 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,696 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:35,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:35,697 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:35,697 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9690614e2b174b38898d8efe515fbb27, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=36.1 K 2024-11-26T14:15:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,698 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9690614e2b174b38898d8efe515fbb27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732630531395 2024-11-26T14:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,699 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ea066104b64849acb933838d7a2d84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732630532864 2024-11-26T14:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,700 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5658de34b0024718a5c32784e6fc7809, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,737 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:35,738 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/bc4a4585897d4509a20e84336370d0aa is 50, key is test_row_0/C:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741997_1173 (size=12813) 2024-11-26T14:15:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,819 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/bc4a4585897d4509a20e84336370d0aa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/bc4a4585897d4509a20e84336370d0aa 2024-11-26T14:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,829 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into bc4a4585897d4509a20e84336370d0aa(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:35,829 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:35,829 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630535551; duration=0sec 2024-11-26T14:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,830 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:35,830 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bc2da3914fea4fe98b1afd83104a379a_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_1/A:col10/1732630535746/Put/seqid=0 2024-11-26T14:15:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741998_1174 (size=14994) 2024-11-26T14:15:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:36,037 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#140 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:36,038 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/271ec48d6c8141ba91dfad220e5c1a5c is 175, key is test_row_0/A:col10/1732630534143/Put/seqid=0 2024-11-26T14:15:36,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1732630536018, max timestamp=1732630536050Num uniques 859; 2024-11-26T14:15:36,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8194, suffix heap size=117064,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=179 2024-11-26T14:15:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741999_1175 (size=31767) 2024-11-26T14:15:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630596084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630596084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630596086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630596088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T14:15:36,155 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:36,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-26T14:15:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:36,172 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:36,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:36,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:36,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630596205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630596205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630596205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630596228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:36,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:36,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bc2da3914fea4fe98b1afd83104a379a_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc2da3914fea4fe98b1afd83104a379a_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:36,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/308da62fb00e42979b2ac6ddc51f7e17, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:36,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/308da62fb00e42979b2ac6ddc51f7e17 is 175, key is test_row_1/A:col10/1732630535746/Put/seqid=0 2024-11-26T14:15:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630596373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,376 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742000_1176 (size=39945) 2024-11-26T14:15:36,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630596417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630596417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630596420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630596445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:36,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:36,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,523 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/271ec48d6c8141ba91dfad220e5c1a5c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c 2024-11-26T14:15:36,552 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 271ec48d6c8141ba91dfad220e5c1a5c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:36,552 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:36,552 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630535550; duration=0sec 2024-11-26T14:15:36,553 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:36,553 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:36,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630596734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630596735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630596736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630596755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:36,772 INFO [master/19945b63c2b4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T14:15:36,772 INFO [master/19945b63c2b4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T14:15:36,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/308da62fb00e42979b2ac6ddc51f7e17 2024-11-26T14:15:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:36,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/87ad7366d38e4c9b9b996a74e959405c is 50, key is test_row_1/B:col10/1732630535746/Put/seqid=0 2024-11-26T14:15:36,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742001_1177 (size=9857) 2024-11-26T14:15:36,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:36,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:36,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:36,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630597254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630597254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630597254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:37,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/87ad7366d38e4c9b9b996a74e959405c 2024-11-26T14:15:37,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630597274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:37,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c96567989bc24332a8ac2be8e78ac32b is 50, key is test_row_1/C:col10/1732630535746/Put/seqid=0 2024-11-26T14:15:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742002_1178 (size=9857) 2024-11-26T14:15:37,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c96567989bc24332a8ac2be8e78ac32b 2024-11-26T14:15:37,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/308da62fb00e42979b2ac6ddc51f7e17 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17 2024-11-26T14:15:37,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17, entries=200, sequenceid=407, filesize=39.0 K 2024-11-26T14:15:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/87ad7366d38e4c9b9b996a74e959405c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c 2024-11-26T14:15:37,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:37,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c, entries=100, sequenceid=407, filesize=9.6 K 2024-11-26T14:15:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c96567989bc24332a8ac2be8e78ac32b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b 2024-11-26T14:15:37,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b, entries=100, sequenceid=407, filesize=9.6 K 2024-11-26T14:15:37,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=130.61 KB/133740 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1684ms, sequenceid=407, compaction requested=false 2024-11-26T14:15:37,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:37,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:37,622 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=130.61 KB heapSize=383.67 KB 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:37,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261a6b36b288154bbe8970c91f3fec6348_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630536083/Put/seqid=0 2024-11-26T14:15:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742003_1179 (size=12454) 2024-11-26T14:15:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:37,718 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261a6b36b288154bbe8970c91f3fec6348_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a6b36b288154bbe8970c91f3fec6348_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:37,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4209a7b1573247ad99063b5ca1fdc852, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:37,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4209a7b1573247ad99063b5ca1fdc852 is 175, key is test_row_0/A:col10/1732630536083/Put/seqid=0 2024-11-26T14:15:37,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742004_1180 (size=31255) 2024-11-26T14:15:38,180 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4209a7b1573247ad99063b5ca1fdc852 2024-11-26T14:15:38,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b79c9e5280c4a49921ed9c937235ea3 is 50, key is test_row_0/B:col10/1732630536083/Put/seqid=0 2024-11-26T14:15:38,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742005_1181 (size=12301) 2024-11-26T14:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:38,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630598294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630598306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630598306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630598308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630598383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,385 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630598413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630598423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630598423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630598424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630598624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630598633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630598643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630598643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,652 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b79c9e5280c4a49921ed9c937235ea3 2024-11-26T14:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:38,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/eef49de1b69243ad9b652ff1ed91603c is 50, key is test_row_0/C:col10/1732630536083/Put/seqid=0 2024-11-26T14:15:38,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742006_1182 (size=12301) 2024-11-26T14:15:38,686 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.26 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/eef49de1b69243ad9b652ff1ed91603c 2024-11-26T14:15:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4209a7b1573247ad99063b5ca1fdc852 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852 2024-11-26T14:15:38,737 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852, entries=150, sequenceid=436, filesize=30.5 K 2024-11-26T14:15:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/8b79c9e5280c4a49921ed9c937235ea3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3 2024-11-26T14:15:38,775 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3, entries=150, sequenceid=436, filesize=12.0 K 2024-11-26T14:15:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/eef49de1b69243ad9b652ff1ed91603c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c 2024-11-26T14:15:38,796 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c, entries=150, sequenceid=436, filesize=12.0 K 2024-11-26T14:15:38,797 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~130.61 KB/133740, heapSize ~383.63 KB/392832, currentSize=53.67 KB/54960 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1175ms, sequenceid=436, compaction requested=true 2024-11-26T14:15:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-26T14:15:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-26T14:15:38,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T14:15:38,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-26T14:15:38,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.6600 sec 2024-11-26T14:15:38,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:38,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6c7f2612f4c46fb81ba1922c3b48f3f_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:38,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742007_1183 (size=14994) 2024-11-26T14:15:38,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:38,996 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6c7f2612f4c46fb81ba1922c3b48f3f_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6c7f2612f4c46fb81ba1922c3b48f3f_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:38,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/81c8095e63c44c1abe1cce5d9c62bd71, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:38,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/81c8095e63c44c1abe1cce5d9c62bd71 is 175, key is test_row_0/A:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630598992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630598995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630598997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630599000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742008_1184 (size=39949) 2024-11-26T14:15:39,038 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/81c8095e63c44c1abe1cce5d9c62bd71 2024-11-26T14:15:39,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5929eac4a9064e2a9a89a0561bbef229 is 50, key is test_row_0/B:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742009_1185 (size=12301) 2024-11-26T14:15:39,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5929eac4a9064e2a9a89a0561bbef229 2024-11-26T14:15:39,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630599115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630599115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630599115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630599115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3ae472a3dd694924bf60fa106c5cd836 is 50, key is test_row_0/C:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742010_1186 (size=12301) 2024-11-26T14:15:39,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3ae472a3dd694924bf60fa106c5cd836 2024-11-26T14:15:39,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/81c8095e63c44c1abe1cce5d9c62bd71 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71 2024-11-26T14:15:39,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71, entries=200, sequenceid=448, filesize=39.0 K 2024-11-26T14:15:39,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5929eac4a9064e2a9a89a0561bbef229 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229 2024-11-26T14:15:39,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229, entries=150, sequenceid=448, filesize=12.0 K 2024-11-26T14:15:39,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3ae472a3dd694924bf60fa106c5cd836 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836 2024-11-26T14:15:39,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836, entries=150, sequenceid=448, filesize=12.0 K 2024-11-26T14:15:39,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 332ms, sequenceid=448, compaction requested=true 2024-11-26T14:15:39,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T14:15:39,272 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:39,275 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:39,275 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:39,275 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:39,275 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:39,275 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=139.6 K 2024-11-26T14:15:39,275 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:39,275 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71] 2024-11-26T14:15:39,277 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271ec48d6c8141ba91dfad220e5c1a5c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:39,277 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308da62fb00e42979b2ac6ddc51f7e17, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732630535621 2024-11-26T14:15:39,278 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4209a7b1573247ad99063b5ca1fdc852, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732630536045 2024-11-26T14:15:39,278 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c8095e63c44c1abe1cce5d9c62bd71, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:39,279 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47272 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:39,279 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:39,279 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:39,279 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/bc4a4585897d4509a20e84336370d0aa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=46.2 K 2024-11-26T14:15:39,279 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4a4585897d4509a20e84336370d0aa, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:39,282 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c96567989bc24332a8ac2be8e78ac32b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732630535746 2024-11-26T14:15:39,282 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting eef49de1b69243ad9b652ff1ed91603c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732630536045 2024-11-26T14:15:39,283 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae472a3dd694924bf60fa106c5cd836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:39,297 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:39,302 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:39,302 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/e272f571cf0d4b839007a8dadd1a8fdc is 50, key is test_row_0/C:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,314 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411267e33d70445fb4894aad3812d8f47687e_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:39,317 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411267e33d70445fb4894aad3812d8f47687e_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:39,317 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267e33d70445fb4894aad3812d8f47687e_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:39,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630538986, max timestamp=1732630539326Num uniques 859; 2024-11-26T14:15:39,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630538986, max timestamp=1732630539326Num uniques 859; 2024-11-26T14:15:39,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630538986, max timestamp=1732630539326Num uniques 859; 2024-11-26T14:15:39,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-26T14:15:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:39,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742011_1187 (size=12949) 2024-11-26T14:15:39,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b1c3df8259c445148c978c0a1b3b2f93_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630538998/Put/seqid=0 2024-11-26T14:15:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742012_1188 (size=4469) 2024-11-26T14:15:39,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630539365, max timestamp=1732630539385Num uniques 859; 2024-11-26T14:15:39,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742013_1189 (size=17534) 2024-11-26T14:15:39,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630539365, max timestamp=1732630539385Num uniques 859; 2024-11-26T14:15:39,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:39,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630539365, max timestamp=1732630539385Num uniques 859; 2024-11-26T14:15:39,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:15:39,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b1c3df8259c445148c978c0a1b3b2f93_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b1c3df8259c445148c978c0a1b3b2f93_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:39,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a3b702619efe4acc9488e310172990e6, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:39,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a3b702619efe4acc9488e310172990e6 is 175, key is test_row_0/A:col10/1732630538998/Put/seqid=0 2024-11-26T14:15:39,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742014_1190 (size=48639) 2024-11-26T14:15:39,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732630539413, max timestamp=1732630539438Num uniques 1018; 2024-11-26T14:15:39,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=19138, suffix heap size=133840,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=418 2024-11-26T14:15:39,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630539413, max timestamp=1732630539438Num uniques 1018; 2024-11-26T14:15:39,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-26T14:15:39,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732630539413, max timestamp=1732630539438Num uniques 1018; 2024-11-26T14:15:39,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=19138, suffix heap size=133840,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=418 2024-11-26T14:15:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630599465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630599469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630599470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630599471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630599573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630599583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630599593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630599593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,756 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/e272f571cf0d4b839007a8dadd1a8fdc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e272f571cf0d4b839007a8dadd1a8fdc 2024-11-26T14:15:39,764 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into e272f571cf0d4b839007a8dadd1a8fdc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:39,764 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:39,764 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=12, startTime=1732630539272; duration=0sec 2024-11-26T14:15:39,764 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:39,764 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:39,764 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:39,767 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47272 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:39,767 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:39,767 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:39,767 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1ecd6c19c4114db98b7c5f2bca9a556a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=46.2 K 2024-11-26T14:15:39,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ecd6c19c4114db98b7c5f2bca9a556a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732630534143 2024-11-26T14:15:39,770 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ad7366d38e4c9b9b996a74e959405c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732630535746 2024-11-26T14:15:39,771 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b79c9e5280c4a49921ed9c937235ea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732630536045 2024-11-26T14:15:39,771 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5929eac4a9064e2a9a89a0561bbef229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:39,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630599783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630599792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,799 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#152 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:39,799 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/52d045307d9346649dbef3376da943c4 is 175, key is test_row_0/A:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,801 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:39,801 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/6ea20ebbed654e179fc5df0fcb12f510 is 50, key is test_row_0/B:col10/1732630538298/Put/seqid=0 2024-11-26T14:15:39,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742015_1191 (size=31903) 2024-11-26T14:15:39,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a3b702619efe4acc9488e310172990e6 2024-11-26T14:15:39,809 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/52d045307d9346649dbef3376da943c4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4 2024-11-26T14:15:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630599803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630599805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742016_1192 (size=12949) 2024-11-26T14:15:39,820 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 52d045307d9346649dbef3376da943c4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:39,820 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:39,821 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=12, startTime=1732630539272; duration=0sec 2024-11-26T14:15:39,821 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:39,821 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:39,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e7401469953f454fb11066ce4e59de32 is 50, key is test_row_0/B:col10/1732630538998/Put/seqid=0 2024-11-26T14:15:39,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742017_1193 (size=12301) 2024-11-26T14:15:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630600093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630600103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630600122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630600123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,228 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/6ea20ebbed654e179fc5df0fcb12f510 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6ea20ebbed654e179fc5df0fcb12f510 2024-11-26T14:15:40,235 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 6ea20ebbed654e179fc5df0fcb12f510(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:40,235 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:40,235 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=12, startTime=1732630539272; duration=0sec 2024-11-26T14:15:40,235 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:40,235 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:40,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e7401469953f454fb11066ce4e59de32 2024-11-26T14:15:40,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 is 50, key is test_row_0/C:col10/1732630538998/Put/seqid=0 2024-11-26T14:15:40,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742018_1194 (size=12301) 2024-11-26T14:15:40,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 2024-11-26T14:15:40,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/a3b702619efe4acc9488e310172990e6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6 2024-11-26T14:15:40,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6, entries=250, sequenceid=470, filesize=47.5 K 2024-11-26T14:15:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e7401469953f454fb11066ce4e59de32 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32 2024-11-26T14:15:40,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32, entries=150, sequenceid=470, filesize=12.0 K 2024-11-26T14:15:40,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 2024-11-26T14:15:40,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47, entries=150, sequenceid=470, filesize=12.0 K 2024-11-26T14:15:40,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=155.74 KB/159482 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1012ms, sequenceid=470, compaction requested=false 2024-11-26T14:15:40,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T14:15:40,352 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:40,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-26T14:15:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:40,364 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:40,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:40,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:40,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T14:15:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:40,538 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=155.74 KB heapSize=433.09 KB 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ba92bf15c1a24a19ac1d2fd1c6b36fe7_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630539468/Put/seqid=0 2024-11-26T14:15:40,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:40,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742019_1195 (size=12454) 2024-11-26T14:15:40,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630600633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630600643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630600653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630600656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630600763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630600764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630600764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630600973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630600973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630600973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:41,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:41,052 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ba92bf15c1a24a19ac1d2fd1c6b36fe7_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba92bf15c1a24a19ac1d2fd1c6b36fe7_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:41,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0867e41adcc487d8a61fd4fb6da3ca2, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:41,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0867e41adcc487d8a61fd4fb6da3ca2 is 175, key is test_row_0/A:col10/1732630539468/Put/seqid=0 2024-11-26T14:15:41,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742020_1196 (size=31255) 2024-11-26T14:15:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630601284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630601285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630601286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,475 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=527, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0867e41adcc487d8a61fd4fb6da3ca2 2024-11-26T14:15:41,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5c7549b5755b435ebd00975071e19491 is 50, key is test_row_0/B:col10/1732630539468/Put/seqid=0 2024-11-26T14:15:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742021_1197 (size=12301) 2024-11-26T14:15:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:41,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630601654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,656 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630601802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630601803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630601803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:41,889 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5c7549b5755b435ebd00975071e19491 2024-11-26T14:15:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5550a95b1d2247a58720aa562c740674 is 50, key is test_row_0/C:col10/1732630539468/Put/seqid=0 2024-11-26T14:15:41,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742022_1198 (size=12301) 2024-11-26T14:15:42,301 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5550a95b1d2247a58720aa562c740674 2024-11-26T14:15:42,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0867e41adcc487d8a61fd4fb6da3ca2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2 2024-11-26T14:15:42,309 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2, entries=150, sequenceid=527, filesize=30.5 K 2024-11-26T14:15:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/5c7549b5755b435ebd00975071e19491 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491 2024-11-26T14:15:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,316 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T14:15:42,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/5550a95b1d2247a58720aa562c740674 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674 2024-11-26T14:15:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,322 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674, entries=150, sequenceid=527, filesize=12.0 K 2024-11-26T14:15:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,323 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~155.74 KB/159482, heapSize ~433.05 KB/443440, currentSize=26.84 KB/27480 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1786ms, sequenceid=527, compaction requested=true 2024-11-26T14:15:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-26T14:15:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-26T14:15:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-26T14:15:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-26T14:15:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9710 sec 2024-11-26T14:15:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:42,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265b86e51b16b44af09bdbf274ba0b5f65_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742023_1199 (size=17534) 2024-11-26T14:15:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T14:15:42,513 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:42,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T14:15:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:42,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:42,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:42,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:42,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630542496, max timestamp=1732630542517Num uniques 859; 2024-11-26T14:15:42,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:42,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630542496, max timestamp=1732630542517Num uniques 859; 2024-11-26T14:15:42,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:42,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630542496, max timestamp=1732630542517Num uniques 859; 2024-11-26T14:15:42,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:15:42,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630602607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:42,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630602723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630602813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,817 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630602823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630602823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,824 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,825 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:42,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:42,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:42,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265b86e51b16b44af09bdbf274ba0b5f65_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b86e51b16b44af09bdbf274ba0b5f65_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:42,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/cda6ba16e91f45478bb3a8285b45b5fb, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:42,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/cda6ba16e91f45478bb3a8285b45b5fb is 175, key is test_row_0/A:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742024_1200 (size=48639) 2024-11-26T14:15:42,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630602933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:42,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:42,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:43,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630603243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:43,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=537, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/cda6ba16e91f45478bb3a8285b45b5fb 2024-11-26T14:15:43,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/bb100ebd6ca14b9b8e821b15f294391f is 50, key is test_row_0/B:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:43,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742025_1201 (size=12301) 2024-11-26T14:15:43,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/bb100ebd6ca14b9b8e821b15f294391f 2024-11-26T14:15:43,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/36d3c3b1eb034ac78593d4f8bdfebf70 is 50, key is test_row_0/C:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:43,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742026_1202 (size=12301) 2024-11-26T14:15:43,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:43,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52636 deadline: 1732630603673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:43,676 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:43,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:43,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/36d3c3b1eb034ac78593d4f8bdfebf70 2024-11-26T14:15:43,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/cda6ba16e91f45478bb3a8285b45b5fb as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb 2024-11-26T14:15:43,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb, entries=250, sequenceid=537, filesize=47.5 K 2024-11-26T14:15:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630603753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:43,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/bb100ebd6ca14b9b8e821b15f294391f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f 2024-11-26T14:15:43,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f, entries=150, sequenceid=537, filesize=12.0 K 2024-11-26T14:15:43,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/36d3c3b1eb034ac78593d4f8bdfebf70 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70 2024-11-26T14:15:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:43,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70, entries=150, sequenceid=537, filesize=12.0 K 2024-11-26T14:15:43,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1357ms, sequenceid=537, compaction requested=true 2024-11-26T14:15:43,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:43,779 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:43,779 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160436 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:43,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,781 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,781 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=156.7 K 2024-11-26T14:15:43,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6ea20ebbed654e179fc5df0fcb12f510, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=48.7 K 2024-11-26T14:15:43,781 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb] 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea20ebbed654e179fc5df0fcb12f510, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d045307d9346649dbef3376da943c4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:43,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e7401469953f454fb11066ce4e59de32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732630538991 2024-11-26T14:15:43,782 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b702619efe4acc9488e310172990e6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732630538986 2024-11-26T14:15:43,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7549b5755b435ebd00975071e19491, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732630539468 2024-11-26T14:15:43,782 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0867e41adcc487d8a61fd4fb6da3ca2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732630539468 2024-11-26T14:15:43,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting bb100ebd6ca14b9b8e821b15f294391f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540642 2024-11-26T14:15:43,782 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda6ba16e91f45478bb3a8285b45b5fb, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540614 2024-11-26T14:15:43,790 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:43,793 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#164 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:43,793 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/265be6b13c6e44a7829a76d941029e20 is 50, key is test_row_0/B:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:43,797 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126cf70f5bad1f44aaa88ce8ececf235024_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:43,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742027_1203 (size=13085) 2024-11-26T14:15:43,799 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126cf70f5bad1f44aaa88ce8ececf235024_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:43,799 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cf70f5bad1f44aaa88ce8ececf235024_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:43,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742028_1204 (size=4469) 2024-11-26T14:15:43,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T14:15:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:43,902 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:43,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:43,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a1464cbe90b64224a68baa9ea47136e0_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630542576/Put/seqid=0 2024-11-26T14:15:43,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742029_1205 (size=12454) 2024-11-26T14:15:44,203 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/265be6b13c6e44a7829a76d941029e20 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/265be6b13c6e44a7829a76d941029e20 2024-11-26T14:15:44,204 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:44,205 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0cf2c47909c4f219d7d9f0fcaf0cda6 is 175, key is test_row_0/A:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:44,209 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 265be6b13c6e44a7829a76d941029e20(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:44,209 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:44,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742030_1206 (size=32039) 2024-11-26T14:15:44,209 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=12, startTime=1732630543779; duration=0sec 2024-11-26T14:15:44,210 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:44,210 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:44,210 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:44,213 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:44,214 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:44,214 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:44,215 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e272f571cf0d4b839007a8dadd1a8fdc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=48.7 K 2024-11-26T14:15:44,216 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e272f571cf0d4b839007a8dadd1a8fdc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732630538298 2024-11-26T14:15:44,216 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8b4c3c8b654bcca68ba07e1ea8ed47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732630538991 2024-11-26T14:15:44,217 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5550a95b1d2247a58720aa562c740674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1732630539468 2024-11-26T14:15:44,217 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d3c3b1eb034ac78593d4f8bdfebf70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540642 2024-11-26T14:15:44,233 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:44,234 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c86f9ad83a9a4645858ff1887520d967 is 50, key is test_row_0/C:col10/1732630540642/Put/seqid=0 2024-11-26T14:15:44,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742031_1207 (size=13085) 2024-11-26T14:15:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a1464cbe90b64224a68baa9ea47136e0_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1464cbe90b64224a68baa9ea47136e0_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/7e1ffa3132f24472828571c3c893d312, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:44,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/7e1ffa3132f24472828571c3c893d312 is 175, key is test_row_0/A:col10/1732630542576/Put/seqid=0 2024-11-26T14:15:44,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742032_1208 (size=31255) 2024-11-26T14:15:44,614 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/c0cf2c47909c4f219d7d9f0fcaf0cda6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6 2024-11-26T14:15:44,618 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into c0cf2c47909c4f219d7d9f0fcaf0cda6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:44,618 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:44,619 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=12, startTime=1732630543778; duration=0sec 2024-11-26T14:15:44,619 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:44,619 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:44,672 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c86f9ad83a9a4645858ff1887520d967 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c86f9ad83a9a4645858ff1887520d967 2024-11-26T14:15:44,676 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into c86f9ad83a9a4645858ff1887520d967(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:44,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:44,677 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=12, startTime=1732630543779; duration=0sec 2024-11-26T14:15:44,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:44,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:44,731 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=573, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/7e1ffa3132f24472828571c3c893d312 2024-11-26T14:15:44,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 is 50, key is test_row_0/B:col10/1732630542576/Put/seqid=0 2024-11-26T14:15:44,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742033_1209 (size=12301) 2024-11-26T14:15:44,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. as already flushing 2024-11-26T14:15:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630604812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52664 deadline: 1732630604843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:44,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52618 deadline: 1732630604843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:44,845 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,845 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:44,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52626 deadline: 1732630604852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:44,860 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in fe0ec8e3e6af878b1c6bcf46977c8bf2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:44,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630604923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:45,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630605133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:45,143 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 2024-11-26T14:15:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c39ddf7997374567b5f52e873738fd56 is 50, key is test_row_0/C:col10/1732630542576/Put/seqid=0 2024-11-26T14:15:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742034_1210 (size=12301) 2024-11-26T14:15:45,158 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=573 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c39ddf7997374567b5f52e873738fd56 2024-11-26T14:15:45,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/7e1ffa3132f24472828571c3c893d312 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312 2024-11-26T14:15:45,168 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312, entries=150, sequenceid=573, filesize=30.5 K 2024-11-26T14:15:45,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 2024-11-26T14:15:45,174 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9, entries=150, sequenceid=573, filesize=12.0 K 2024-11-26T14:15:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/c39ddf7997374567b5f52e873738fd56 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56 2024-11-26T14:15:45,180 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56, entries=150, sequenceid=573, filesize=12.0 K 2024-11-26T14:15:45,181 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1278ms, sequenceid=573, compaction requested=false 2024-11-26T14:15:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T14:15:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T14:15:45,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T14:15:45,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-11-26T14:15:45,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6700 sec 2024-11-26T14:15:45,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:45,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:45,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660c3d0c7ccdf4e1f9bcd4e1e22185407_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:45,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742035_1211 (size=12454) 2024-11-26T14:15:45,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630545505, max timestamp=1732630545549Num uniques 859; 2024-11-26T14:15:45,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:15:45,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630545505, max timestamp=1732630545549Num uniques 859; 2024-11-26T14:15:45,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630545531, max timestamp=1732630545549Num uniques 859; 2024-11-26T14:15:45,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:15:45,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:45,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630605626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:45,661 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-11-26T14:15:45,661 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:45,661 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,661 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:45,663 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-26T14:15:45,663 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:45,663 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,663 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:45,665 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-11-26T14:15:45,665 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:45,665 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,666 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:45,673 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-26T14:15:45,673 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:45,674 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,674 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:45,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:45,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630605743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:45,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:45,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660c3d0c7ccdf4e1f9bcd4e1e22185407_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660c3d0c7ccdf4e1f9bcd4e1e22185407_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:45,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4c264d2b4db0419cacf15633cc1044c1, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:45,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4c264d2b4db0419cacf15633cc1044c1 is 175, key is test_row_0/A:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:45,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742036_1212 (size=31255) 2024-11-26T14:15:45,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630605953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:46,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630606263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:46,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:46,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=587, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4c264d2b4db0419cacf15633cc1044c1 2024-11-26T14:15:46,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e66f6649156d499ab3dd5927d2c9822b is 50, key is test_row_0/B:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:46,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742037_1213 (size=12301) 2024-11-26T14:15:46,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e66f6649156d499ab3dd5927d2c9822b 2024-11-26T14:15:46,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/4257ab49737649c08cf72604d0a0c953 is 50, key is test_row_0/C:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742038_1214 (size=12301) 2024-11-26T14:15:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T14:15:46,683 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:46,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/4257ab49737649c08cf72604d0a0c953 2024-11-26T14:15:46,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/4c264d2b4db0419cacf15633cc1044c1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1 2024-11-26T14:15:46,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1, entries=150, sequenceid=587, filesize=30.5 K 2024-11-26T14:15:46,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/e66f6649156d499ab3dd5927d2c9822b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b 2024-11-26T14:15:46,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b, entries=150, sequenceid=587, filesize=12.0 K 2024-11-26T14:15:46,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/4257ab49737649c08cf72604d0a0c953 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953 2024-11-26T14:15:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52668 deadline: 1732630606773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:46,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:15:46,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953, entries=150, sequenceid=587, filesize=12.0 K 2024-11-26T14:15:46,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1333ms, sequenceid=587, compaction requested=true 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:46,778 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe0ec8e3e6af878b1c6bcf46977c8bf2:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:46,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:46,778 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:46,779 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:46,780 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/A is initiating minor compaction (all files) 2024-11-26T14:15:46,780 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/A in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:46,780 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=92.3 K 2024-11-26T14:15:46,780 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:46,780 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1] 2024-11-26T14:15:46,780 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0cf2c47909c4f219d7d9f0fcaf0cda6, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540642 2024-11-26T14:15:46,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:46,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1ffa3132f24472828571c3c893d312, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1732630542576 2024-11-26T14:15:46,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/B is initiating minor compaction (all files) 2024-11-26T14:15:46,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/B in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:46,781 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/265be6b13c6e44a7829a76d941029e20, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=36.8 K 2024-11-26T14:15:46,781 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c264d2b4db0419cacf15633cc1044c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630544802 2024-11-26T14:15:46,781 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 265be6b13c6e44a7829a76d941029e20, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540642 2024-11-26T14:15:46,784 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f7d6adefdc4aefb38312d2ae4ef7a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1732630542576 2024-11-26T14:15:46,784 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e66f6649156d499ab3dd5927d2c9822b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630544802 2024-11-26T14:15:46,817 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:46,826 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#B#compaction#174 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:46,826 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/6cc5fdc78ce947009f992b6404f09e7e is 50, key is test_row_0/B:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:46,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a168f633734b4d35b3d64ce5a874d787_fe0ec8e3e6af878b1c6bcf46977c8bf2 store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:46,859 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a168f633734b4d35b3d64ce5a874d787_fe0ec8e3e6af878b1c6bcf46977c8bf2, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:46,859 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a168f633734b4d35b3d64ce5a874d787_fe0ec8e3e6af878b1c6bcf46977c8bf2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:46,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742040_1216 (size=4469) 2024-11-26T14:15:46,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742039_1215 (size=13187) 2024-11-26T14:15:46,869 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#A#compaction#173 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:46,869 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/287b89b8b88c4da7ae603168eb74af5d is 175, key is test_row_0/A:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:46,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742041_1217 (size=32141) 2024-11-26T14:15:46,895 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/287b89b8b88c4da7ae603168eb74af5d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/287b89b8b88c4da7ae603168eb74af5d 2024-11-26T14:15:46,903 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/A of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 287b89b8b88c4da7ae603168eb74af5d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:46,903 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:46,903 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/A, priority=13, startTime=1732630546778; duration=0sec 2024-11-26T14:15:46,903 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:46,903 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:A 2024-11-26T14:15:46,904 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:46,905 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:46,905 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): fe0ec8e3e6af878b1c6bcf46977c8bf2/C is initiating minor compaction (all files) 2024-11-26T14:15:46,905 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe0ec8e3e6af878b1c6bcf46977c8bf2/C in TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:46,905 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c86f9ad83a9a4645858ff1887520d967, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp, totalSize=36.8 K 2024-11-26T14:15:46,906 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86f9ad83a9a4645858ff1887520d967, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732630540642 2024-11-26T14:15:46,907 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39ddf7997374567b5f52e873738fd56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=573, earliestPutTs=1732630542576 2024-11-26T14:15:46,909 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4257ab49737649c08cf72604d0a0c953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630544802 2024-11-26T14:15:46,924 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe0ec8e3e6af878b1c6bcf46977c8bf2#C#compaction#175 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:46,925 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/546b38dcfe2747ccbe76d12e93fd3fdf is 50, key is test_row_0/C:col10/1732630544810/Put/seqid=0 2024-11-26T14:15:46,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742042_1218 (size=13187) 2024-11-26T14:15:46,949 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/546b38dcfe2747ccbe76d12e93fd3fdf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/546b38dcfe2747ccbe76d12e93fd3fdf 2024-11-26T14:15:46,954 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/C of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 546b38dcfe2747ccbe76d12e93fd3fdf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:46,954 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:46,954 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/C, priority=13, startTime=1732630546778; duration=0sec 2024-11-26T14:15:46,954 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:46,954 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:C 2024-11-26T14:15:47,273 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/6cc5fdc78ce947009f992b6404f09e7e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6cc5fdc78ce947009f992b6404f09e7e 2024-11-26T14:15:47,279 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe0ec8e3e6af878b1c6bcf46977c8bf2/B of fe0ec8e3e6af878b1c6bcf46977c8bf2 into 6cc5fdc78ce947009f992b6404f09e7e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:47,279 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:47,279 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., storeName=fe0ec8e3e6af878b1c6bcf46977c8bf2/B, priority=13, startTime=1732630546778; duration=0sec 2024-11-26T14:15:47,279 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:47,280 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe0ec8e3e6af878b1c6bcf46977c8bf2:B 2024-11-26T14:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:47,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-26T14:15:47,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:47,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:47,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:47,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:47,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:47,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:47,700 INFO [Thread-570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-570. 2024-11-26T14:15:47,701 DEBUG [Thread-570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:47,701 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:47,701 DEBUG [Thread-570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:47,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:47,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126441689d3f4d54926a5b2644e1409ecbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630545617/Put/seqid=0 2024-11-26T14:15:47,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742043_1219 (size=12454) 2024-11-26T14:15:47,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:47,795 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-26T14:15:47,795 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:47,795 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:47,795 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:47,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:47,810 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126441689d3f4d54926a5b2644e1409ecbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126441689d3f4d54926a5b2644e1409ecbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:47,812 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/80a69750f8114d1b903bfd157e13b6d8, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:47,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/80a69750f8114d1b903bfd157e13b6d8 is 175, key is test_row_0/A:col10/1732630545617/Put/seqid=0 2024-11-26T14:15:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742044_1220 (size=31255) 2024-11-26T14:15:47,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=626, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/80a69750f8114d1b903bfd157e13b6d8 2024-11-26T14:15:47,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/73868e5f389a4e3d8de2aa04c35cf3c0 is 50, key is test_row_0/B:col10/1732630545617/Put/seqid=0 2024-11-26T14:15:47,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742045_1221 (size=12301) 2024-11-26T14:15:48,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/73868e5f389a4e3d8de2aa04c35cf3c0 2024-11-26T14:15:48,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/95b2eed0034c4999818c8cd16a46ee5c is 50, key is test_row_0/C:col10/1732630545617/Put/seqid=0 2024-11-26T14:15:48,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742046_1222 (size=12301) 2024-11-26T14:15:48,385 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:15:48,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/95b2eed0034c4999818c8cd16a46ee5c 2024-11-26T14:15:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/80a69750f8114d1b903bfd157e13b6d8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/80a69750f8114d1b903bfd157e13b6d8 2024-11-26T14:15:48,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/80a69750f8114d1b903bfd157e13b6d8, entries=150, sequenceid=626, filesize=30.5 K 2024-11-26T14:15:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/73868e5f389a4e3d8de2aa04c35cf3c0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/73868e5f389a4e3d8de2aa04c35cf3c0 2024-11-26T14:15:48,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/73868e5f389a4e3d8de2aa04c35cf3c0, entries=150, sequenceid=626, filesize=12.0 K 2024-11-26T14:15:48,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/95b2eed0034c4999818c8cd16a46ee5c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/95b2eed0034c4999818c8cd16a46ee5c 2024-11-26T14:15:48,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/95b2eed0034c4999818c8cd16a46ee5c, entries=150, sequenceid=626, filesize=12.0 K 2024-11-26T14:15:48,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=6.71 KB/6870 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 1048ms, sequenceid=626, compaction requested=false 2024-11-26T14:15:48,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: 2024-11-26T14:15:48,855 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-26T14:15:48,855 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:48,855 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,855 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:48,874 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-26T14:15:48,874 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:48,874 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,874 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:48,894 INFO [Thread-572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-572. 2024-11-26T14:15:48,895 DEBUG [Thread-572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:15:48,895 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,895 DEBUG [Thread-572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1714 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1723 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 764 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2291 rows 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 763 2024-11-26T14:15:48,895 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2289 rows 2024-11-26T14:15:48,895 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:15:48,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:15:48,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:48,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:15:48,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:15:48,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:48,899 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630548899"}]},"ts":"1732630548899"} 2024-11-26T14:15:48,901 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:15:48,901 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:15:48,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:15:48,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, UNASSIGN}] 2024-11-26T14:15:48,903 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, UNASSIGN 2024-11-26T14:15:48,904 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:48,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, UNASSIGN because future has completed 2024-11-26T14:15:48,907 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:15:48,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:49,060 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(122): Close fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:49,060 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:15:49,060 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1722): Closing fe0ec8e3e6af878b1c6bcf46977c8bf2, disabling compactions & flushes 2024-11-26T14:15:49,060 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:49,060 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:49,060 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. after waiting 0 ms 2024-11-26T14:15:49,060 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:49,060 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2902): Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=A 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=B 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe0ec8e3e6af878b1c6bcf46977c8bf2, store=C 2024-11-26T14:15:49,061 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:49,066 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ecbbe2d9999b4f4dbe3fb488d8a8d5a1_fe0ec8e3e6af878b1c6bcf46977c8bf2 is 50, key is test_row_0/A:col10/1732630548853/Put/seqid=0 2024-11-26T14:15:49,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742047_1223 (size=9914) 2024-11-26T14:15:49,070 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:49,074 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ecbbe2d9999b4f4dbe3fb488d8a8d5a1_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ecbbe2d9999b4f4dbe3fb488d8a8d5a1_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:49,075 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/9b19a51fc793450b9f93ae34b7fd8420, store: [table=TestAcidGuarantees family=A region=fe0ec8e3e6af878b1c6bcf46977c8bf2] 2024-11-26T14:15:49,075 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/9b19a51fc793450b9f93ae34b7fd8420 is 175, key is test_row_0/A:col10/1732630548853/Put/seqid=0 2024-11-26T14:15:49,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742048_1224 (size=22561) 2024-11-26T14:15:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:49,479 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=633, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/9b19a51fc793450b9f93ae34b7fd8420 2024-11-26T14:15:49,488 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/1d26b264e16c4b529bc512566fee1f37 is 50, key is test_row_0/B:col10/1732630548853/Put/seqid=0 2024-11-26T14:15:49,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742049_1225 (size=9857) 2024-11-26T14:15:49,499 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/1d26b264e16c4b529bc512566fee1f37 2024-11-26T14:15:49,507 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3a44604d0b6c4c62adfe47e00c964881 is 50, key is test_row_0/C:col10/1732630548853/Put/seqid=0 2024-11-26T14:15:49,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742050_1226 (size=9857) 2024-11-26T14:15:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:49,918 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3a44604d0b6c4c62adfe47e00c964881 2024-11-26T14:15:49,924 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/A/9b19a51fc793450b9f93ae34b7fd8420 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/9b19a51fc793450b9f93ae34b7fd8420 2024-11-26T14:15:49,929 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/9b19a51fc793450b9f93ae34b7fd8420, entries=100, sequenceid=633, filesize=22.0 K 2024-11-26T14:15:49,930 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/B/1d26b264e16c4b529bc512566fee1f37 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1d26b264e16c4b529bc512566fee1f37 2024-11-26T14:15:49,936 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1d26b264e16c4b529bc512566fee1f37, entries=100, sequenceid=633, filesize=9.6 K 2024-11-26T14:15:49,937 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/.tmp/C/3a44604d0b6c4c62adfe47e00c964881 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3a44604d0b6c4c62adfe47e00c964881 2024-11-26T14:15:49,943 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3a44604d0b6c4c62adfe47e00c964881, entries=100, sequenceid=633, filesize=9.6 K 2024-11-26T14:15:49,944 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 884ms, sequenceid=633, compaction requested=true 2024-11-26T14:15:49,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1] to archive 2024-11-26T14:15:49,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:49,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/afb576ef3a6c45f2bb0a5946ac4b3a02 2024-11-26T14:15:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4723168d8ae0437894c53491d982f376 2024-11-26T14:15:49,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/8277290acd254ee99a9918c88d9836cc 2024-11-26T14:15:50,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/001794d65a7b493c86507d52311f7e74 2024-11-26T14:15:50,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/18e018d36bc04947b1fc29059ca458f6 2024-11-26T14:15:50,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/68131853465d43b288e414757c53a205 2024-11-26T14:15:50,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e04babee93154eb4ae3366fcc701a02a 2024-11-26T14:15:50,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/2ecd4715b2ab42268a569c4c48f28eab 2024-11-26T14:15:50,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5b34f236de54429f8d6870b5f25dc89b 2024-11-26T14:15:50,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a1f7499fbebd40b6ae3e3ef53d01babf 2024-11-26T14:15:50,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/5663a29b684944e1b2af80ff642b0488 2024-11-26T14:15:50,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c1bbcb32915f41dab2cd36c9c35c7480 2024-11-26T14:15:50,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/e232c00900174aef8f8c3895a8cfeaa8 2024-11-26T14:15:50,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a64861e1adaa48b490075988c3749346 2024-11-26T14:15:50,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/271ec48d6c8141ba91dfad220e5c1a5c 2024-11-26T14:15:50,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/20007f61623f4d5fb7b41b6e1bcaab43 2024-11-26T14:15:50,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/308da62fb00e42979b2ac6ddc51f7e17 2024-11-26T14:15:50,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4209a7b1573247ad99063b5ca1fdc852 2024-11-26T14:15:50,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/81c8095e63c44c1abe1cce5d9c62bd71 2024-11-26T14:15:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:50,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/52d045307d9346649dbef3376da943c4 2024-11-26T14:15:50,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/a3b702619efe4acc9488e310172990e6 2024-11-26T14:15:50,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0867e41adcc487d8a61fd4fb6da3ca2 2024-11-26T14:15:50,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/cda6ba16e91f45478bb3a8285b45b5fb 2024-11-26T14:15:50,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/c0cf2c47909c4f219d7d9f0fcaf0cda6 2024-11-26T14:15:50,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/7e1ffa3132f24472828571c3c893d312 2024-11-26T14:15:50,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/4c264d2b4db0419cacf15633cc1044c1 2024-11-26T14:15:50,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/9586b51232aa4c8aab4645c67c4126f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4e44a73ea89a4f87ab557cd2fe6ca7db, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4906bb66c1534e879131960f032437cd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fe044ad57cbe4be1b3a322e4e4d92661, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1ecd6c19c4114db98b7c5f2bca9a556a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6ea20ebbed654e179fc5df0fcb12f510, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/265be6b13c6e44a7829a76d941029e20, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b] to archive 2024-11-26T14:15:50,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:50,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b7433e7efb74b39ae4d1a1d09d95b5e 2024-11-26T14:15:50,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/10499d3217e541c0adbba06251a37373 2024-11-26T14:15:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/9586b51232aa4c8aab4645c67c4126f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/9586b51232aa4c8aab4645c67c4126f7 2024-11-26T14:15:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/0605ce06fa5046d0aad4dbb06996c63a 2024-11-26T14:15:50,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f76bfa76c06a4ba5832bd7ec36f1d14d 2024-11-26T14:15:50,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4e44a73ea89a4f87ab557cd2fe6ca7db to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4e44a73ea89a4f87ab557cd2fe6ca7db 2024-11-26T14:15:50,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/f77fa7b3b67844fabb9950706af1838a 2024-11-26T14:15:50,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5b701cf3a3494bd1af1cc17023c42161 2024-11-26T14:15:50,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4906bb66c1534e879131960f032437cd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/4906bb66c1534e879131960f032437cd 2024-11-26T14:15:50,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fed66327fd594b579097b329b9b0da0e 2024-11-26T14:15:50,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fcaf00fc316745fe8e506ad8dea92831 2024-11-26T14:15:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fe044ad57cbe4be1b3a322e4e4d92661 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/fe044ad57cbe4be1b3a322e4e4d92661 2024-11-26T14:15:50,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/16f6e55ae63a45f5b8d416b47ac5ad93 2024-11-26T14:15:50,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/469a032d8a5045d59880d55c265a4e45 2024-11-26T14:15:50,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1ecd6c19c4114db98b7c5f2bca9a556a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1ecd6c19c4114db98b7c5f2bca9a556a 2024-11-26T14:15:50,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/378d35eda33b4d2689aadbdc5d9db5b8 2024-11-26T14:15:50,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/87ad7366d38e4c9b9b996a74e959405c 2024-11-26T14:15:50,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/8b79c9e5280c4a49921ed9c937235ea3 2024-11-26T14:15:50,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6ea20ebbed654e179fc5df0fcb12f510 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6ea20ebbed654e179fc5df0fcb12f510 2024-11-26T14:15:50,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5929eac4a9064e2a9a89a0561bbef229 2024-11-26T14:15:50,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e7401469953f454fb11066ce4e59de32 2024-11-26T14:15:50,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/5c7549b5755b435ebd00975071e19491 2024-11-26T14:15:50,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/265be6b13c6e44a7829a76d941029e20 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/265be6b13c6e44a7829a76d941029e20 2024-11-26T14:15:50,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/bb100ebd6ca14b9b8e821b15f294391f 2024-11-26T14:15:50,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/b1f7d6adefdc4aefb38312d2ae4ef7a9 2024-11-26T14:15:50,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/e66f6649156d499ab3dd5927d2c9822b 2024-11-26T14:15:50,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/82f63ebc469e481f8b6236e34e51fce4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/28227833f475455189b0d2c95e38dff1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/178f5daa65da42e4bfb5c200162cd485, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9690614e2b174b38898d8efe515fbb27, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/bc4a4585897d4509a20e84336370d0aa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e272f571cf0d4b839007a8dadd1a8fdc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c86f9ad83a9a4645858ff1887520d967, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953] to archive 2024-11-26T14:15:50,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:15:50,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/b7d33dc4f26a4baf8b3b7ca492210ee9 2024-11-26T14:15:50,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa018d38cbb44cd99eec408ea1989d5a 2024-11-26T14:15:50,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/82f63ebc469e481f8b6236e34e51fce4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/82f63ebc469e481f8b6236e34e51fce4 2024-11-26T14:15:50,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/7b69253d453245bb9dfd9f4e7864a901 2024-11-26T14:15:50,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/794075aa655b44c8a0809b7f6d1a5c5c 2024-11-26T14:15:50,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/28227833f475455189b0d2c95e38dff1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/28227833f475455189b0d2c95e38dff1 2024-11-26T14:15:50,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/de4a26dc9ee945218254a0e0b734efc2 2024-11-26T14:15:50,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/849fb12528db444da564fbf709f557dc 2024-11-26T14:15:50,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/178f5daa65da42e4bfb5c200162cd485 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/178f5daa65da42e4bfb5c200162cd485 2024-11-26T14:15:50,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e0ff1669dd5e485a92485f4a74d793be 2024-11-26T14:15:50,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/a9a678b4448244df89f2376ac19ff93d 2024-11-26T14:15:50,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9690614e2b174b38898d8efe515fbb27 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9690614e2b174b38898d8efe515fbb27 2024-11-26T14:15:50,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/aa55db99a82e4b9c9e51d0018c50f15c 2024-11-26T14:15:50,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/84ea066104b64849acb933838d7a2d84 2024-11-26T14:15:50,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/bc4a4585897d4509a20e84336370d0aa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/bc4a4585897d4509a20e84336370d0aa 2024-11-26T14:15:50,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5658de34b0024718a5c32784e6fc7809 2024-11-26T14:15:50,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c96567989bc24332a8ac2be8e78ac32b 2024-11-26T14:15:50,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/eef49de1b69243ad9b652ff1ed91603c 2024-11-26T14:15:50,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e272f571cf0d4b839007a8dadd1a8fdc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/e272f571cf0d4b839007a8dadd1a8fdc 2024-11-26T14:15:50,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3ae472a3dd694924bf60fa106c5cd836 2024-11-26T14:15:50,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/9c8b4c3c8b654bcca68ba07e1ea8ed47 2024-11-26T14:15:50,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/5550a95b1d2247a58720aa562c740674 2024-11-26T14:15:50,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c86f9ad83a9a4645858ff1887520d967 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c86f9ad83a9a4645858ff1887520d967 2024-11-26T14:15:50,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/36d3c3b1eb034ac78593d4f8bdfebf70 2024-11-26T14:15:50,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/c39ddf7997374567b5f52e873738fd56 2024-11-26T14:15:50,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/4257ab49737649c08cf72604d0a0c953 2024-11-26T14:15:50,232 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits/636.seqid, newMaxSeqId=636, maxSeqId=4 2024-11-26T14:15:50,234 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. 2024-11-26T14:15:50,234 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1676): Region close journal for fe0ec8e3e6af878b1c6bcf46977c8bf2: Waiting for close lock at 1732630549060Running coprocessor pre-close hooks at 1732630549060Disabling compacts and flushes for region at 1732630549060Disabling writes for close at 1732630549060Obtaining lock to block concurrent updates at 1732630549060Preparing flush snapshotting stores in fe0ec8e3e6af878b1c6bcf46977c8bf2 at 1732630549060Finished memstore snapshotting TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732630549061 (+1 ms)Flushing stores of TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2. at 1732630549061Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/A: creating writer at 1732630549061Flushing mob file fe0ec8e3e6af878b1c6bcf46977c8bf2/A: appending metadata at 1732630549065 (+4 ms)Flushing mob file fe0ec8e3e6af878b1c6bcf46977c8bf2/A: closing flushed file at 1732630549066 (+1 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/A: appending metadata at 1732630549075 (+9 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/A: closing flushed file at 1732630549075Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/B: creating writer at 1732630549485 (+410 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/B: appending metadata at 1732630549487 (+2 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/B: closing flushed file at 1732630549487Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/C: creating writer at 1732630549503 (+16 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/C: appending metadata at 1732630549506 (+3 ms)Flushing fe0ec8e3e6af878b1c6bcf46977c8bf2/C: closing flushed file at 1732630549506Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1400c750: reopening flushed file at 1732630549923 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7164e220: reopening flushed file at 1732630549929 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c49b0f1: reopening flushed file at 1732630549936 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe0ec8e3e6af878b1c6bcf46977c8bf2 in 884ms, sequenceid=633, compaction requested=true at 1732630549944 (+8 ms)Writing region close event to WAL at 1732630550220 (+276 ms)Running coprocessor post-close hooks at 1732630550233 (+13 ms)Closed at 1732630550234 (+1 ms) 2024-11-26T14:15:50,243 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(157): Closed fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:50,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=fe0ec8e3e6af878b1c6bcf46977c8bf2, regionState=CLOSED 2024-11-26T14:15:50,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:50,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T14:15:50,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseRegionProcedure fe0ec8e3e6af878b1c6bcf46977c8bf2, server=19945b63c2b4,40025,1732630490995 in 1.3490 sec 2024-11-26T14:15:50,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-11-26T14:15:50,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe0ec8e3e6af878b1c6bcf46977c8bf2, UNASSIGN in 1.3550 sec 2024-11-26T14:15:50,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T14:15:50,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3600 sec 2024-11-26T14:15:50,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630550265"}]},"ts":"1732630550265"} 2024-11-26T14:15:50,268 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:15:50,269 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:15:50,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3740 sec 2024-11-26T14:15:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T14:15:51,053 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:51,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:15:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,055 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T14:15:51,056 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=48, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,059 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,061 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits] 2024-11-26T14:15:51,064 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/287b89b8b88c4da7ae603168eb74af5d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/287b89b8b88c4da7ae603168eb74af5d 2024-11-26T14:15:51,065 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/80a69750f8114d1b903bfd157e13b6d8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/80a69750f8114d1b903bfd157e13b6d8 2024-11-26T14:15:51,067 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/9b19a51fc793450b9f93ae34b7fd8420 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/A/9b19a51fc793450b9f93ae34b7fd8420 2024-11-26T14:15:51,069 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1d26b264e16c4b529bc512566fee1f37 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/1d26b264e16c4b529bc512566fee1f37 2024-11-26T14:15:51,071 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6cc5fdc78ce947009f992b6404f09e7e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/6cc5fdc78ce947009f992b6404f09e7e 2024-11-26T14:15:51,072 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/73868e5f389a4e3d8de2aa04c35cf3c0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/B/73868e5f389a4e3d8de2aa04c35cf3c0 2024-11-26T14:15:51,075 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3a44604d0b6c4c62adfe47e00c964881 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/3a44604d0b6c4c62adfe47e00c964881 2024-11-26T14:15:51,076 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/546b38dcfe2747ccbe76d12e93fd3fdf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/546b38dcfe2747ccbe76d12e93fd3fdf 2024-11-26T14:15:51,077 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/95b2eed0034c4999818c8cd16a46ee5c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/C/95b2eed0034c4999818c8cd16a46ee5c 2024-11-26T14:15:51,080 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits/636.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2/recovered.edits/636.seqid 2024-11-26T14:15:51,081 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,081 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:15:51,082 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:15:51,082 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T14:15:51,088 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a6b36b288154bbe8970c91f3fec6348_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a6b36b288154bbe8970c91f3fec6348_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,090 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261f2a023edbc14b5dba024ffd8ed678e3_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261f2a023edbc14b5dba024ffd8ed678e3_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,094 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262555395c54604a8c9f41cf5ca9d2c7d4_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262555395c54604a8c9f41cf5ca9d2c7d4_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,096 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c0823de3cf14662bdd81d8c6e653a8c_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262c0823de3cf14662bdd81d8c6e653a8c_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,098 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126441689d3f4d54926a5b2644e1409ecbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126441689d3f4d54926a5b2644e1409ecbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,099 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cacf4c2904c43b4803b347248a72fbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cacf4c2904c43b4803b347248a72fbf_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,101 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cf83af3704941f78edc6b2de5a24ed0_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264cf83af3704941f78edc6b2de5a24ed0_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,105 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b86e51b16b44af09bdbf274ba0b5f65_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265b86e51b16b44af09bdbf274ba0b5f65_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,107 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660c3d0c7ccdf4e1f9bcd4e1e22185407_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660c3d0c7ccdf4e1f9bcd4e1e22185407_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,110 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266cf5dd9703724c72997d5eb3172e6ec8_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266cf5dd9703724c72997d5eb3172e6ec8_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,111 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112672c9aea619bc4bc29ee9b39e63d8876a_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112672c9aea619bc4bc29ee9b39e63d8876a_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,113 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126852b027235564db2880366e4ee4a44fe_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126852b027235564db2880366e4ee4a44fe_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,114 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1464cbe90b64224a68baa9ea47136e0_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a1464cbe90b64224a68baa9ea47136e0_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,116 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a91ce2e00c5c4dc897eeb0a11e2f90c3_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a91ce2e00c5c4dc897eeb0a11e2f90c3_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,118 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b1c3df8259c445148c978c0a1b3b2f93_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b1c3df8259c445148c978c0a1b3b2f93_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,120 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba92bf15c1a24a19ac1d2fd1c6b36fe7_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ba92bf15c1a24a19ac1d2fd1c6b36fe7_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc2da3914fea4fe98b1afd83104a379a_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc2da3914fea4fe98b1afd83104a379a_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6c7f2612f4c46fb81ba1922c3b48f3f_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6c7f2612f4c46fb81ba1922c3b48f3f_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e207db4f25bb4f87a50aa12ab4a76038_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e207db4f25bb4f87a50aa12ab4a76038_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,128 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2720ae9200744dfa360b1a633ffbde8_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e2720ae9200744dfa360b1a633ffbde8_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ecbbe2d9999b4f4dbe3fb488d8a8d5a1_fe0ec8e3e6af878b1c6bcf46977c8bf2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ecbbe2d9999b4f4dbe3fb488d8a8d5a1_fe0ec8e3e6af878b1c6bcf46977c8bf2 2024-11-26T14:15:51,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:15:51,136 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=48, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,141 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:15:51,147 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:15:51,152 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=48, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,152 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:15:51,153 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630551152"}]},"ts":"9223372036854775807"} 2024-11-26T14:15:51,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:15:51,158 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe0ec8e3e6af878b1c6bcf46977c8bf2, NAME => 'TestAcidGuarantees,,1732630523470.fe0ec8e3e6af878b1c6bcf46977c8bf2.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:15:51,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:15:51,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630551159"}]},"ts":"9223372036854775807"} 2024-11-26T14:15:51,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:15:51,164 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=48, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T14:15:51,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 111 msec 2024-11-26T14:15:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T14:15:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:15:51,374 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:51,384 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_289023253_22 at /127.0.0.1:59880 [Waiting for operation #218] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_289023253_22 at /127.0.0.1:37000 [Waiting for operation #667] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1171789504_22 at /127.0.0.1:37374 [Waiting for operation #443] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1171789504_22 at /127.0.0.1:37100 [Waiting for operation #614] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1092 (was 973) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3672 (was 3898) 2024-11-26T14:15:51,395 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1092, ProcessCount=11, AvailableMemoryMB=3672 2024-11-26T14:15:51,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:15:51,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:15:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:15:51,399 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:15:51,399 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:51,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 49 2024-11-26T14:15:51,400 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:15:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-26T14:15:51,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742051_1227 (size=1061) 2024-11-26T14:15:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-26T14:15:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-26T14:15:51,814 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:15:51,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742052_1228 (size=53) 2024-11-26T14:15:51,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:15:51,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 66833f8312ba730f0c055c76a8c04f74, disabling compactions & flushes 2024-11-26T14:15:51,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:51,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:51,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. after waiting 0 ms 2024-11-26T14:15:51,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:51,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:51,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66833f8312ba730f0c055c76a8c04f74: Waiting for close lock at 1732630551831Disabling compacts and flushes for region at 1732630551831Disabling writes for close at 1732630551832 (+1 ms)Writing region close event to WAL at 1732630551832Closed at 1732630551832 2024-11-26T14:15:51,833 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:15:51,833 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630551833"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630551833"}]},"ts":"1732630551833"} 2024-11-26T14:15:51,835 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:15:51,836 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:15:51,836 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630551836"}]},"ts":"1732630551836"} 2024-11-26T14:15:51,838 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:15:51,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, ASSIGN}] 2024-11-26T14:15:51,840 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, ASSIGN 2024-11-26T14:15:51,841 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:15:51,992 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=66833f8312ba730f0c055c76a8c04f74, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:51,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, ASSIGN because future has completed 2024-11-26T14:15:51,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:15:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-26T14:15:52,150 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,150 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:15:52,150 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,150 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:15:52,150 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,150 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,151 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,153 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:52,153 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66833f8312ba730f0c055c76a8c04f74 columnFamilyName A 2024-11-26T14:15:52,153 DEBUG [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:52,153 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(327): Store=66833f8312ba730f0c055c76a8c04f74/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:52,153 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,155 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:52,155 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66833f8312ba730f0c055c76a8c04f74 columnFamilyName B 2024-11-26T14:15:52,155 DEBUG [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:52,156 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(327): Store=66833f8312ba730f0c055c76a8c04f74/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:52,156 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,157 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:15:52,157 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66833f8312ba730f0c055c76a8c04f74 columnFamilyName C 2024-11-26T14:15:52,157 DEBUG [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:15:52,158 INFO [StoreOpener-66833f8312ba730f0c055c76a8c04f74-1 {}] regionserver.HStore(327): Store=66833f8312ba730f0c055c76a8c04f74/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:15:52,158 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,158 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,158 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,159 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,160 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,160 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,160 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:15:52,161 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,164 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:15:52,164 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 66833f8312ba730f0c055c76a8c04f74; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71876539, jitterRate=0.07104389369487762}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:15:52,164 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,165 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 66833f8312ba730f0c055c76a8c04f74: Running coprocessor pre-open hook at 1732630552150Writing region info on filesystem at 1732630552150Initializing all the Stores at 1732630552151 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630552151Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630552151Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630552151Cleaning up temporary data from old regions at 1732630552160 (+9 ms)Running coprocessor post-open hooks at 1732630552164 (+4 ms)Region opened successfully at 1732630552165 (+1 ms) 2024-11-26T14:15:52,166 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., pid=51, masterSystemTime=1732630552147 2024-11-26T14:15:52,167 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,168 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,168 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=66833f8312ba730f0c055c76a8c04f74, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:15:52,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T14:15:52,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; OpenRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 in 176 msec 2024-11-26T14:15:52,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-26T14:15:52,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, ASSIGN in 332 msec 2024-11-26T14:15:52,174 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:15:52,174 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630552174"}]},"ts":"1732630552174"} 2024-11-26T14:15:52,176 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:15:52,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:15:52,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 780 msec 2024-11-26T14:15:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-26T14:15:52,533 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:52,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4adb97a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743412a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,538 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:15:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:15:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76a17bbf 2024-11-26T14:15:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:15:52,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:15:52,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf5d01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c464fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c067da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,555 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3913987f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,560 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee9f95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c961dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,567 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36698ed4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e82210d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,571 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374d1288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d10bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,575 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3de8a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5654019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75db1a1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8ab030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4697a994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee03a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eba4b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b80abb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:15:52,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:15:52,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fdbbd86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:15:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:15:52,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7382fdd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:15:52,613 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,613 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,621 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:52,622 DEBUG [Thread-1088 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,622 DEBUG [Thread-1088 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,622 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,623 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,624 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,625 DEBUG [Thread-1079 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,625 DEBUG [Thread-1079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,626 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-26T14:15:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:52,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,636 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,640 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,642 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:52,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:52,643 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,643 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:15:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:15:52,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:15:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:52,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:15:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:52,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:52,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:52,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:52,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:15:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:15:52,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e634d20d10a644e3a6acd8c94f3bdb06 is 50, key is test_row_0/A:col10/1732630552644/Put/seqid=0 2024-11-26T14:15:52,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742053_1229 (size=12001) 2024-11-26T14:15:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:52,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e634d20d10a644e3a6acd8c94f3bdb06 2024-11-26T14:15:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630612730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630552699, max timestamp=1732630552737Num uniques 859; 2024-11-26T14:15:52,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630612732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630552699, max timestamp=1732630552737Num uniques 859; 2024-11-26T14:15:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:52,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630552699, max timestamp=1732630552737Num uniques 859; 2024-11-26T14:15:52,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:15:52,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:52,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630612802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630612809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630612820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/e838c80501ee40108d02edd54dd17f64 is 50, key is test_row_0/B:col10/1732630552644/Put/seqid=0 2024-11-26T14:15:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630612853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630612853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742054_1230 (size=12001) 2024-11-26T14:15:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630612923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630612923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630612935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:52,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:52,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630613068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630613068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630613144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630613144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630613153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:53,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/e838c80501ee40108d02edd54dd17f64 2024-11-26T14:15:53,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/d8548b9a7cdc485d91842c24c80dfddb is 50, key is test_row_0/C:col10/1732630552644/Put/seqid=0 2024-11-26T14:15:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630613383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630613396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742055_1231 (size=12001) 2024-11-26T14:15:53,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/d8548b9a7cdc485d91842c24c80dfddb 2024-11-26T14:15:53,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e634d20d10a644e3a6acd8c94f3bdb06 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06 2024-11-26T14:15:53,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:15:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/e838c80501ee40108d02edd54dd17f64 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64 2024-11-26T14:15:53,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:15:53,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/d8548b9a7cdc485d91842c24c80dfddb as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb 2024-11-26T14:15:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630613454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630613454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:15:53,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for 66833f8312ba730f0c055c76a8c04f74 in 815ms, sequenceid=11, compaction requested=false 2024-11-26T14:15:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:53,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630552784, max timestamp=1732630553470Num uniques 1009; 2024-11-26T14:15:53,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:15:53,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:53,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=102.08 KB heapSize=290.38 KB 2024-11-26T14:15:53,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:53,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:53,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:15:53,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/4bf3e811efaa4737a06c767388787418 is 50, key is test_row_0/A:col10/1732630553470/Put/seqid=0 2024-11-26T14:15:53,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742056_1232 (size=12001) 2024-11-26T14:15:53,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/4bf3e811efaa4737a06c767388787418 2024-11-26T14:15:53,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/d447c483a31d4b22a80734a051eeed5a is 50, key is test_row_0/B:col10/1732630553470/Put/seqid=0 2024-11-26T14:15:53,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742057_1233 (size=12001) 2024-11-26T14:15:53,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/d447c483a31d4b22a80734a051eeed5a 2024-11-26T14:15:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630613589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fb85ae58aea3484db2b6442580c629fa is 50, key is test_row_0/C:col10/1732630553470/Put/seqid=0 2024-11-26T14:15:53,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742058_1234 (size=12001) 2024-11-26T14:15:53,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630613705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630613903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:53,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:53,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630613916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630613923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630613964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630613964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fb85ae58aea3484db2b6442580c629fa 2024-11-26T14:15:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:54,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:54,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/4bf3e811efaa4737a06c767388787418 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418 2024-11-26T14:15:54,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:15:54,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/d447c483a31d4b22a80734a051eeed5a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a 2024-11-26T14:15:54,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:15:54,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fb85ae58aea3484db2b6442580c629fa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa 2024-11-26T14:15:54,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:15:54,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.08 KB/104530, heapSize ~290.33 KB/297296, currentSize=80.51 KB/82440 for 66833f8312ba730f0c055c76a8c04f74 in 707ms, sequenceid=49, compaction requested=false 2024-11-26T14:15:54,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:54,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T14:15:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:54,244 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:54,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/7099b6202cf4491e93b42fc9d0e44077 is 50, key is test_row_0/A:col10/1732630554236/Put/seqid=0 2024-11-26T14:15:54,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742059_1235 (size=14341) 2024-11-26T14:15:54,335 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/7099b6202cf4491e93b42fc9d0e44077 2024-11-26T14:15:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/67fd56a9f4504ff5857eea2ede57e7e0 is 50, key is test_row_0/B:col10/1732630554236/Put/seqid=0 2024-11-26T14:15:54,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742060_1236 (size=12001) 2024-11-26T14:15:54,420 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/67fd56a9f4504ff5857eea2ede57e7e0 2024-11-26T14:15:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630614431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/381c0c8f495b4b79aa50451876b2e554 is 50, key is test_row_0/C:col10/1732630554236/Put/seqid=0 2024-11-26T14:15:54,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742061_1237 (size=12001) 2024-11-26T14:15:54,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630614546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630614753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:54,920 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/381c0c8f495b4b79aa50451876b2e554 2024-11-26T14:15:54,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/7099b6202cf4491e93b42fc9d0e44077 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077 2024-11-26T14:15:54,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630614937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,948 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 66833f8312ba730f0c055c76a8c04f74 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630614924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,952 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 66833f8312ba730f0c055c76a8c04f74 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:54,977 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077, entries=200, sequenceid=65, filesize=14.0 K 2024-11-26T14:15:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/67fd56a9f4504ff5857eea2ede57e7e0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0 2024-11-26T14:15:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630614984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:54,997 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 66833f8312ba730f0c055c76a8c04f74 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:55,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630614992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:55,004 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 66833f8312ba730f0c055c76a8c04f74 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:55,018 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0, entries=150, sequenceid=65, filesize=11.7 K 2024-11-26T14:15:55,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/381c0c8f495b4b79aa50451876b2e554 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554 2024-11-26T14:15:55,033 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554, entries=150, sequenceid=65, filesize=11.7 K 2024-11-26T14:15:55,037 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 66833f8312ba730f0c055c76a8c04f74 in 794ms, sequenceid=65, compaction requested=true 2024-11-26T14:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-26T14:15:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-26T14:15:55,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T14:15:55,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4070 sec 2024-11-26T14:15:55,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4400 sec 2024-11-26T14:15:55,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:55,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:55,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:55,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:55,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:55,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:55,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:55,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/8327f3d45c034401ae2938fe71fb04a8 is 50, key is test_row_0/A:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742062_1238 (size=14341) 2024-11-26T14:15:55,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/8327f3d45c034401ae2938fe71fb04a8 2024-11-26T14:15:55,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 is 50, key is test_row_0/B:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630615222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742063_1239 (size=12001) 2024-11-26T14:15:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630615336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630615554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:55,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 2024-11-26T14:15:55,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f0a6228dd0d2413abd871066d4b40e02 is 50, key is test_row_0/C:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742064_1240 (size=12001) 2024-11-26T14:15:55,794 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:15:55,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630615876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:56,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f0a6228dd0d2413abd871066d4b40e02 2024-11-26T14:15:56,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/8327f3d45c034401ae2938fe71fb04a8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8 2024-11-26T14:15:56,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8, entries=200, sequenceid=82, filesize=14.0 K 2024-11-26T14:15:56,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 2024-11-26T14:15:56,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T14:15:56,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f0a6228dd0d2413abd871066d4b40e02 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02 2024-11-26T14:15:56,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T14:15:56,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 66833f8312ba730f0c055c76a8c04f74 in 1123ms, sequenceid=82, compaction requested=true 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:56,210 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:56,210 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:56,213 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:56,213 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:15:56,213 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:56,213 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=51.4 K 2024-11-26T14:15:56,214 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e634d20d10a644e3a6acd8c94f3bdb06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630552644 2024-11-26T14:15:56,214 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf3e811efaa4737a06c767388787418, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630552784 2024-11-26T14:15:56,215 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7099b6202cf4491e93b42fc9d0e44077, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630553529 2024-11-26T14:15:56,215 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8327f3d45c034401ae2938fe71fb04a8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554356 2024-11-26T14:15:56,248 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:56,248 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:15:56,248 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:56,248 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=46.9 K 2024-11-26T14:15:56,249 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e838c80501ee40108d02edd54dd17f64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630552644 2024-11-26T14:15:56,249 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d447c483a31d4b22a80734a051eeed5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630552784 2024-11-26T14:15:56,250 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fd56a9f4504ff5857eea2ede57e7e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630553529 2024-11-26T14:15:56,251 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e40c92e8d204a15bfaf6f6c5d6e33d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554385 2024-11-26T14:15:56,275 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:56,276 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/93f9723af75446bab2a4b3928cd9ec1d is 50, key is test_row_0/A:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:56,295 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:56,296 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/9434fc992d8f4907a6f657a16e2daacd is 50, key is test_row_0/B:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:56,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742065_1241 (size=12139) 2024-11-26T14:15:56,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742066_1242 (size=12139) 2024-11-26T14:15:56,387 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/9434fc992d8f4907a6f657a16e2daacd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/9434fc992d8f4907a6f657a16e2daacd 2024-11-26T14:15:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:56,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:15:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:56,414 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 9434fc992d8f4907a6f657a16e2daacd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:56,414 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:56,414 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=12, startTime=1732630556210; duration=0sec 2024-11-26T14:15:56,415 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:56,415 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:15:56,415 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:15:56,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,421 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:15:56,422 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:15:56,422 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:56,422 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=46.9 K 2024-11-26T14:15:56,427 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d8548b9a7cdc485d91842c24c80dfddb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630552644 2024-11-26T14:15:56,428 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting fb85ae58aea3484db2b6442580c629fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630552784 2024-11-26T14:15:56,435 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 381c0c8f495b4b79aa50451876b2e554, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732630553529 2024-11-26T14:15:56,435 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a6228dd0d2413abd871066d4b40e02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554385 2024-11-26T14:15:56,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/29c69f97687a4b7c9269a1c4d6bb2df2 is 50, key is test_row_0/A:col10/1732630555208/Put/seqid=0 2024-11-26T14:15:56,463 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:56,464 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/95eb48dc9ecb4329861c2aad8dfd219d is 50, key is test_row_0/C:col10/1732630554385/Put/seqid=0 2024-11-26T14:15:56,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742067_1243 (size=14341) 2024-11-26T14:15:56,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/29c69f97687a4b7c9269a1c4d6bb2df2 2024-11-26T14:15:56,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742068_1244 (size=12139) 2024-11-26T14:15:56,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/7d6fdc7f52d04e27b3538e5fc14a9e57 is 50, key is test_row_0/B:col10/1732630555208/Put/seqid=0 2024-11-26T14:15:56,539 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/95eb48dc9ecb4329861c2aad8dfd219d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95eb48dc9ecb4329861c2aad8dfd219d 2024-11-26T14:15:56,547 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into 95eb48dc9ecb4329861c2aad8dfd219d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:56,547 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:56,547 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=12, startTime=1732630556210; duration=0sec 2024-11-26T14:15:56,548 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:56,548 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:15:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742069_1245 (size=12001) 2024-11-26T14:15:56,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/7d6fdc7f52d04e27b3538e5fc14a9e57 2024-11-26T14:15:56,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/ce6288c492df4c9495fbfd56ca044455 is 50, key is test_row_0/C:col10/1732630555208/Put/seqid=0 2024-11-26T14:15:56,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630616632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:56,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742070_1246 (size=12001) 2024-11-26T14:15:56,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/ce6288c492df4c9495fbfd56ca044455 2024-11-26T14:15:56,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/29c69f97687a4b7c9269a1c4d6bb2df2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2 2024-11-26T14:15:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2, entries=200, sequenceid=99, filesize=14.0 K 2024-11-26T14:15:56,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/7d6fdc7f52d04e27b3538e5fc14a9e57 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57 2024-11-26T14:15:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630616757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:56,805 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/93f9723af75446bab2a4b3928cd9ec1d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/93f9723af75446bab2a4b3928cd9ec1d 2024-11-26T14:15:56,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T14:15:56,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/ce6288c492df4c9495fbfd56ca044455 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455 2024-11-26T14:15:56,818 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into 93f9723af75446bab2a4b3928cd9ec1d(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:56,818 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:56,818 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=12, startTime=1732630556210; duration=0sec 2024-11-26T14:15:56,819 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:56,819 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:15:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T14:15:56,820 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:56,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-26T14:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:56,826 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:56,827 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:56,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:56,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455, entries=150, sequenceid=99, filesize=11.7 K 2024-11-26T14:15:56,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 66833f8312ba730f0c055c76a8c04f74 in 434ms, sequenceid=99, compaction requested=false 2024-11-26T14:15:56,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:56,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:56,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T14:15:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:56,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:57,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b53a6758136540acbb9f0a01c5a3b475 is 50, key is test_row_0/A:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742071_1247 (size=12001) 2024-11-26T14:15:57,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630617037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b53a6758136540acbb9f0a01c5a3b475 2024-11-26T14:15:57,048 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 66833f8312ba730f0c055c76a8c04f74 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:15:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630617042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630617042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630617045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630617050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/8257bec1b9a348dcbb520bd6d85a7988 is 50, key is test_row_0/B:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742072_1248 (size=12001) 2024-11-26T14:15:57,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/8257bec1b9a348dcbb520bd6d85a7988 2024-11-26T14:15:57,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/0e97bff1976a422791a4ed4e0c9fac7b is 50, key is test_row_0/C:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:57,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T14:15:57,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:57,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:57,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630617164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630617164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630617164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742073_1249 (size=12001) 2024-11-26T14:15:57,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/0e97bff1976a422791a4ed4e0c9fac7b 2024-11-26T14:15:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630617177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b53a6758136540acbb9f0a01c5a3b475 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475 2024-11-26T14:15:57,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T14:15:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/8257bec1b9a348dcbb520bd6d85a7988 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988 2024-11-26T14:15:57,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T14:15:57,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/0e97bff1976a422791a4ed4e0c9fac7b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b 2024-11-26T14:15:57,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T14:15:57,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 66833f8312ba730f0c055c76a8c04f74 in 273ms, sequenceid=118, compaction requested=true 2024-11-26T14:15:57,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:57,247 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:57,247 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:57,248 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:57,249 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:15:57,249 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,249 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/93f9723af75446bab2a4b3928cd9ec1d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=37.6 K 2024-11-26T14:15:57,249 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f9723af75446bab2a4b3928cd9ec1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554385 2024-11-26T14:15:57,249 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:57,249 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c69f97687a4b7c9269a1c4d6bb2df2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630555158 2024-11-26T14:15:57,249 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:15:57,250 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b53a6758136540acbb9f0a01c5a3b475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:57,250 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,250 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/9434fc992d8f4907a6f657a16e2daacd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.3 K 2024-11-26T14:15:57,251 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9434fc992d8f4907a6f657a16e2daacd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554385 2024-11-26T14:15:57,252 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6fdc7f52d04e27b3538e5fc14a9e57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630555208 2024-11-26T14:15:57,253 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8257bec1b9a348dcbb520bd6d85a7988, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:57,271 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:57,271 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e78c531be35f49fd93f39a0de7694163 is 50, key is test_row_0/A:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,288 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#204 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:57,289 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/4052fef8cbeb4c42970d55f57e994475 is 50, key is test_row_0/B:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T14:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,311 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:57,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742074_1250 (size=12241) 2024-11-26T14:15:57,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b372ac5bef464619a65635b423259b0c is 50, key is test_row_0/A:col10/1732630557045/Put/seqid=0 2024-11-26T14:15:57,340 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e78c531be35f49fd93f39a0de7694163 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e78c531be35f49fd93f39a0de7694163 2024-11-26T14:15:57,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742075_1251 (size=12241) 2024-11-26T14:15:57,378 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into e78c531be35f49fd93f39a0de7694163(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:57,378 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:57,378 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630557246; duration=0sec 2024-11-26T14:15:57,378 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:57,378 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:15:57,378 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:57,385 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:57,385 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:15:57,385 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:57,385 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95eb48dc9ecb4329861c2aad8dfd219d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.3 K 2024-11-26T14:15:57,392 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95eb48dc9ecb4329861c2aad8dfd219d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732630554385 2024-11-26T14:15:57,398 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6288c492df4c9495fbfd56ca044455, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732630555208 2024-11-26T14:15:57,400 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e97bff1976a422791a4ed4e0c9fac7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:57,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742076_1252 (size=12101) 2024-11-26T14:15:57,419 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b372ac5bef464619a65635b423259b0c 2024-11-26T14:15:57,434 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:57,435 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/eed9ef29a66145d6a6ceaac1b41e0acf is 50, key is test_row_0/C:col10/1732630556624/Put/seqid=0 2024-11-26T14:15:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/567464576f974aabb2d3f643d82e37d8 is 50, key is test_row_0/B:col10/1732630557045/Put/seqid=0 2024-11-26T14:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630617449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630617453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630617456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630617457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742077_1253 (size=12241) 2024-11-26T14:15:57,511 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/eed9ef29a66145d6a6ceaac1b41e0acf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/eed9ef29a66145d6a6ceaac1b41e0acf 2024-11-26T14:15:57,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742078_1254 (size=12101) 2024-11-26T14:15:57,518 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into eed9ef29a66145d6a6ceaac1b41e0acf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:57,519 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:57,519 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630557247; duration=0sec 2024-11-26T14:15:57,519 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:57,519 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:15:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630617573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630617573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630617573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630617576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,782 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/4052fef8cbeb4c42970d55f57e994475 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/4052fef8cbeb4c42970d55f57e994475 2024-11-26T14:15:57,789 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 4052fef8cbeb4c42970d55f57e994475(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:57,789 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:57,789 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630557247; duration=0sec 2024-11-26T14:15:57,789 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:57,789 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:15:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630617795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630617795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630617795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630617795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:57,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:57,920 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/567464576f974aabb2d3f643d82e37d8 2024-11-26T14:15:57,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/848b8043b3e64847a5ae3542675ab888 is 50, key is test_row_0/C:col10/1732630557045/Put/seqid=0 2024-11-26T14:15:57,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742079_1255 (size=12101) 2024-11-26T14:15:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:57,964 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/848b8043b3e64847a5ae3542675ab888 2024-11-26T14:15:57,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b372ac5bef464619a65635b423259b0c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c 2024-11-26T14:15:57,993 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-26T14:15:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/567464576f974aabb2d3f643d82e37d8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8 2024-11-26T14:15:58,003 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8, entries=150, sequenceid=133, filesize=11.8 K 2024-11-26T14:15:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/848b8043b3e64847a5ae3542675ab888 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888 2024-11-26T14:15:58,011 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888, entries=150, sequenceid=133, filesize=11.8 K 2024-11-26T14:15:58,012 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 66833f8312ba730f0c055c76a8c04f74 in 701ms, sequenceid=133, compaction requested=false 2024-11-26T14:15:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-26T14:15:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-26T14:15:58,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T14:15:58,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1870 sec 2024-11-26T14:15:58,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2010 sec 2024-11-26T14:15:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:58,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fcfad41990cf490c88613516d304be7e is 50, key is test_row_0/A:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:58,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630618158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630618165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630618177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742080_1256 (size=12151) 2024-11-26T14:15:58,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fcfad41990cf490c88613516d304be7e 2024-11-26T14:15:58,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3403f39a06874043866fe55745f175a1 is 50, key is test_row_0/B:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:58,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742081_1257 (size=12151) 2024-11-26T14:15:58,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3403f39a06874043866fe55745f175a1 2024-11-26T14:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630618285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630618285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/71b077e1e63348ecac5eebd04f3a8a56 is 50, key is test_row_0/C:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630618297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630618303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742082_1258 (size=12151) 2024-11-26T14:15:58,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/71b077e1e63348ecac5eebd04f3a8a56 2024-11-26T14:15:58,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fcfad41990cf490c88613516d304be7e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e 2024-11-26T14:15:58,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3403f39a06874043866fe55745f175a1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1 2024-11-26T14:15:58,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:58,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/71b077e1e63348ecac5eebd04f3a8a56 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56 2024-11-26T14:15:58,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T14:15:58,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 66833f8312ba730f0c055c76a8c04f74 in 297ms, sequenceid=156, compaction requested=true 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:58,422 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:58,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:15:58,423 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:58,424 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:58,425 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:15:58,425 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:58,425 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e78c531be35f49fd93f39a0de7694163, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.6 K 2024-11-26T14:15:58,425 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e78c531be35f49fd93f39a0de7694163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:58,425 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b372ac5bef464619a65635b423259b0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732630557029 2024-11-26T14:15:58,426 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcfad41990cf490c88613516d304be7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:15:58,436 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:58,436 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:15:58,436 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:58,436 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/4052fef8cbeb4c42970d55f57e994475, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.6 K 2024-11-26T14:15:58,438 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:58,439 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/96fafa78d313423a99f4fabc48871c4c is 50, key is test_row_0/A:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:58,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4052fef8cbeb4c42970d55f57e994475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:58,448 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 567464576f974aabb2d3f643d82e37d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732630557029 2024-11-26T14:15:58,452 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3403f39a06874043866fe55745f175a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:15:58,502 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:58,503 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/e01e60c29ce344289fae4785fdbfb923 is 50, key is test_row_0/B:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:58,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742083_1259 (size=12493) 2024-11-26T14:15:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:58,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:58,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e083050c497141c9ad3dd882fcb97a41 is 50, key is test_row_0/A:col10/1732630558509/Put/seqid=0 2024-11-26T14:15:58,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742084_1260 (size=12493) 2024-11-26T14:15:58,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742085_1261 (size=16931) 2024-11-26T14:15:58,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e083050c497141c9ad3dd882fcb97a41 2024-11-26T14:15:58,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/2884d3c4aedf45e58fc398467d506367 is 50, key is test_row_0/B:col10/1732630558509/Put/seqid=0 2024-11-26T14:15:58,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630618601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630618605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630618610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630618611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742086_1262 (size=12151) 2024-11-26T14:15:58,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/2884d3c4aedf45e58fc398467d506367 2024-11-26T14:15:58,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b5d8c8eea11c49d794b3351c9b7f1eaf is 50, key is test_row_0/C:col10/1732630558509/Put/seqid=0 2024-11-26T14:15:58,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742087_1263 (size=12151) 2024-11-26T14:15:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630618723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630618723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630618723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630618723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630618934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630618934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630618937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630618937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:58,948 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/96fafa78d313423a99f4fabc48871c4c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/96fafa78d313423a99f4fabc48871c4c 2024-11-26T14:15:58,981 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into 96fafa78d313423a99f4fabc48871c4c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:58,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:58,981 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630558422; duration=0sec 2024-11-26T14:15:58,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:58,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:15:58,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T14:15:58,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:15:58,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:15:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-26T14:15:58,986 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:15:58,988 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:15:58,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:15:58,988 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/e01e60c29ce344289fae4785fdbfb923 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e01e60c29ce344289fae4785fdbfb923 2024-11-26T14:15:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:15:58,994 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:58,994 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:15:58,994 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:58,994 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/eed9ef29a66145d6a6ceaac1b41e0acf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.6 K 2024-11-26T14:15:58,995 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed9ef29a66145d6a6ceaac1b41e0acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732630556550 2024-11-26T14:15:58,996 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 848b8043b3e64847a5ae3542675ab888, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732630557029 2024-11-26T14:15:59,000 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71b077e1e63348ecac5eebd04f3a8a56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:15:59,009 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into e01e60c29ce344289fae4785fdbfb923(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:59,009 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:59,009 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630558422; duration=0sec 2024-11-26T14:15:59,009 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:59,009 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:15:59,054 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:59,055 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/af16eb9fe61842ad9ae8faf329d3da21 is 50, key is test_row_0/C:col10/1732630557439/Put/seqid=0 2024-11-26T14:15:59,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742088_1264 (size=12493) 2024-11-26T14:15:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:15:59,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b5d8c8eea11c49d794b3351c9b7f1eaf 2024-11-26T14:15:59,108 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/af16eb9fe61842ad9ae8faf329d3da21 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/af16eb9fe61842ad9ae8faf329d3da21 2024-11-26T14:15:59,115 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into af16eb9fe61842ad9ae8faf329d3da21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:15:59,115 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:59,115 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630558422; duration=0sec 2024-11-26T14:15:59,116 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:59,116 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:15:59,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e083050c497141c9ad3dd882fcb97a41 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41 2024-11-26T14:15:59,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41, entries=250, sequenceid=170, filesize=16.5 K 2024-11-26T14:15:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/2884d3c4aedf45e58fc398467d506367 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367 2024-11-26T14:15:59,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T14:15:59,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b5d8c8eea11c49d794b3351c9b7f1eaf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf 2024-11-26T14:15:59,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T14:15:59,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 66833f8312ba730f0c055c76a8c04f74 in 653ms, sequenceid=170, compaction requested=false 2024-11-26T14:15:59,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:59,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:59,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:15:59,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d874cad440214903bc50dfbaf78391d9 is 50, key is test_row_0/A:col10/1732630558605/Put/seqid=0 2024-11-26T14:15:59,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742089_1265 (size=14541) 2024-11-26T14:15:59,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d874cad440214903bc50dfbaf78391d9 2024-11-26T14:15:59,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:15:59,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/15847a3aab644a90992b9cfc37d470ae is 50, key is test_row_0/B:col10/1732630558605/Put/seqid=0 2024-11-26T14:15:59,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630619276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630619334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630619336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630619337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742090_1266 (size=12151) 2024-11-26T14:15:59,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/15847a3aab644a90992b9cfc37d470ae 2024-11-26T14:15:59,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/40261651f4e44852bdf38937488514c8 is 50, key is test_row_0/C:col10/1732630558605/Put/seqid=0 2024-11-26T14:15:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742091_1267 (size=12151) 2024-11-26T14:15:59,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630619442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630619453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630619454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630619456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:15:59,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630619654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630619664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630619663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:15:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630619676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:15:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:15:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:15:59,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:15:59,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/40261651f4e44852bdf38937488514c8 2024-11-26T14:15:59,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d874cad440214903bc50dfbaf78391d9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9 2024-11-26T14:15:59,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9, entries=200, sequenceid=194, filesize=14.2 K 2024-11-26T14:15:59,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/15847a3aab644a90992b9cfc37d470ae as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae 2024-11-26T14:15:59,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae, entries=150, sequenceid=194, filesize=11.9 K 2024-11-26T14:15:59,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/40261651f4e44852bdf38937488514c8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8 2024-11-26T14:15:59,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-26T14:15:59,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 66833f8312ba730f0c055c76a8c04f74 in 627ms, sequenceid=194, compaction requested=true 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:15:59,877 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:59,877 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:15:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:15:59,878 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:59,878 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:15:59,878 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:15:59,878 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:15:59,878 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,878 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,878 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/96fafa78d313423a99f4fabc48871c4c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=42.9 K 2024-11-26T14:15:59,878 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e01e60c29ce344289fae4785fdbfb923, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.9 K 2024-11-26T14:15:59,879 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e01e60c29ce344289fae4785fdbfb923, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:15:59,879 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fafa78d313423a99f4fabc48871c4c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:15:59,880 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e083050c497141c9ad3dd882fcb97a41, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732630558148 2024-11-26T14:15:59,880 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2884d3c4aedf45e58fc398467d506367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732630558175 2024-11-26T14:15:59,881 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d874cad440214903bc50dfbaf78391d9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558591 2024-11-26T14:15:59,881 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 15847a3aab644a90992b9cfc37d470ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558605 2024-11-26T14:15:59,894 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:59,895 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f7005ea15d4d4cc19bbd032d9cff0fe9 is 50, key is test_row_0/B:col10/1732630558605/Put/seqid=0 2024-11-26T14:15:59,905 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:15:59,906 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e118470bc6ae478bb505f4dd1ad689bc is 50, key is test_row_0/A:col10/1732630558605/Put/seqid=0 2024-11-26T14:15:59,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742092_1268 (size=12595) 2024-11-26T14:15:59,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742093_1269 (size=12595) 2024-11-26T14:15:59,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:15:59,939 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:59,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:15:59,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:15:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/f907fa74c55f46faa0cb6c05238d2994 is 50, key is test_row_0/A:col10/1732630559287/Put/seqid=0 2024-11-26T14:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:15:59,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:15:59,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742094_1270 (size=12151) 2024-11-26T14:15:59,994 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/f907fa74c55f46faa0cb6c05238d2994 2024-11-26T14:16:00,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/163a30543de545fe89b29dbaee440893 is 50, key is test_row_0/B:col10/1732630559287/Put/seqid=0 2024-11-26T14:16:00,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742095_1271 (size=12151) 2024-11-26T14:16:00,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630620034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630620041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630620043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630620045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:16:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630620153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630620156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630620163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630620163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,338 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f7005ea15d4d4cc19bbd032d9cff0fe9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f7005ea15d4d4cc19bbd032d9cff0fe9 2024-11-26T14:16:00,349 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/e118470bc6ae478bb505f4dd1ad689bc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e118470bc6ae478bb505f4dd1ad689bc 2024-11-26T14:16:00,350 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into f7005ea15d4d4cc19bbd032d9cff0fe9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:00,350 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:00,350 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630559877; duration=0sec 2024-11-26T14:16:00,351 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:00,351 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:00,351 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:00,354 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:00,354 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:00,354 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:00,354 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/af16eb9fe61842ad9ae8faf329d3da21, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=35.9 K 2024-11-26T14:16:00,354 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting af16eb9fe61842ad9ae8faf329d3da21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732630557439 2024-11-26T14:16:00,355 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d8c8eea11c49d794b3351c9b7f1eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732630558175 2024-11-26T14:16:00,355 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 40261651f4e44852bdf38937488514c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558605 2024-11-26T14:16:00,356 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into e118470bc6ae478bb505f4dd1ad689bc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:00,356 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:00,356 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630559877; duration=0sec 2024-11-26T14:16:00,356 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:00,356 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630620363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630620373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,376 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630620373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,376 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/ce01047b65604de4b19e0e2f91c3ee60 is 50, key is test_row_0/C:col10/1732630558605/Put/seqid=0 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630620373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742096_1272 (size=12595) 2024-11-26T14:16:00,459 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/163a30543de545fe89b29dbaee440893 2024-11-26T14:16:00,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b905ce7fd1784b9e8362ecddd6d3fa7f is 50, key is test_row_0/C:col10/1732630559287/Put/seqid=0 2024-11-26T14:16:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742097_1273 (size=12151) 2024-11-26T14:16:00,551 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b905ce7fd1784b9e8362ecddd6d3fa7f 2024-11-26T14:16:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/f907fa74c55f46faa0cb6c05238d2994 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994 2024-11-26T14:16:00,577 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T14:16:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/163a30543de545fe89b29dbaee440893 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893 2024-11-26T14:16:00,585 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T14:16:00,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b905ce7fd1784b9e8362ecddd6d3fa7f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f 2024-11-26T14:16:00,593 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T14:16:00,603 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 66833f8312ba730f0c055c76a8c04f74 in 664ms, sequenceid=206, compaction requested=false 2024-11-26T14:16:00,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-26T14:16:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-26T14:16:00,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T14:16:00,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-11-26T14:16:00,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6340 sec 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630560021, max timestamp=1732630560673Num uniques 859; 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630560021, max timestamp=1732630560673Num uniques 859; 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630560021, max timestamp=1732630560673Num uniques 859; 2024-11-26T14:16:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:00,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:16:00,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:00,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:00,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/2569bd7acc3e4b378a103b863e8c2a98 is 50, key is test_row_0/A:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742098_1274 (size=14541) 2024-11-26T14:16:00,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630560713, max timestamp=1732630560741Num uniques 859; 2024-11-26T14:16:00,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:16:00,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630560713, max timestamp=1732630560743Num uniques 859; 2024-11-26T14:16:00,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:16:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630560713, max timestamp=1732630560743Num uniques 859; 2024-11-26T14:16:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:16:00,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630560793, max timestamp=1732630560813Num uniques 1118; 2024-11-26T14:16:00,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-26T14:16:00,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630560793, max timestamp=1732630560813Num uniques 1018; 2024-11-26T14:16:00,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T14:16:00,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630560793, max timestamp=1732630560813Num uniques 1018; 2024-11-26T14:16:00,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T14:16:00,863 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/ce01047b65604de4b19e0e2f91c3ee60 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce01047b65604de4b19e0e2f91c3ee60 2024-11-26T14:16:00,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732630560843, max timestamp=1732630560859Num uniques 1077; 2024-11-26T14:16:00,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-26T14:16:00,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732630560843, max timestamp=1732630560859Num uniques 1077; 2024-11-26T14:16:00,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-26T14:16:00,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732630560843, max timestamp=1732630560859Num uniques 1077; 2024-11-26T14:16:00,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-26T14:16:00,906 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into ce01047b65604de4b19e0e2f91c3ee60(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:00,906 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:00,906 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630559877; duration=0sec 2024-11-26T14:16:00,906 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:00,906 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630620923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:00,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630560917, max timestamp=1732630560931Num uniques 1136; 2024-11-26T14:16:00,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-26T14:16:00,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630560920, max timestamp=1732630560931Num uniques 1136; 2024-11-26T14:16:00,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=6870, suffix heap size=146112,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=150 2024-11-26T14:16:00,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630560917, max timestamp=1732630560931Num uniques 1136; 2024-11-26T14:16:00,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-26T14:16:00,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732630560973, max timestamp=1732630560991Num uniques 1095; 2024-11-26T14:16:00,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732630560973, max timestamp=1732630560991Num uniques 1009; 2024-11-26T14:16:00,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T14:16:00,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-26T14:16:00,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732630560973, max timestamp=1732630560991Num uniques 1095; 2024-11-26T14:16:00,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630561039, max timestamp=1732630561058Num uniques 1104; 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630561039, max timestamp=1732630561058Num uniques 1104; 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732630561039, max timestamp=1732630561058Num uniques 1104; 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T14:16:01,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-26T14:16:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732630561086, max timestamp=1732630561100Num uniques 1063; 2024-11-26T14:16:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1732630561086, max timestamp=1732630561100Num uniques 1063; 2024-11-26T14:16:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-26T14:16:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-11-26T14:16:01,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561089, max timestamp=1732630561100Num uniques 1063; 2024-11-26T14:16:01,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=6870, suffix heap size=138520,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=150 2024-11-26T14:16:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T14:16:01,133 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:01,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-26T14:16:01,136 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:01,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/2569bd7acc3e4b378a103b863e8c2a98 2024-11-26T14:16:01,137 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:01,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:01,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732630561129, max timestamp=1732630561136Num uniques 1122; 2024-11-26T14:16:01,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-11-26T14:16:01,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732630561129, max timestamp=1732630561136Num uniques 1122; 2024-11-26T14:16:01,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=12452, suffix heap size=144656,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=272 2024-11-26T14:16:01,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1732630561129, max timestamp=1732630561136Num uniques 1009; 2024-11-26T14:16:01,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12452, suffix heap size=132904,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=272 2024-11-26T14:16:01,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/90ed99b67da046da9be83a082ec6dd26 is 50, key is test_row_0/B:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742099_1275 (size=12151) 2024-11-26T14:16:01,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1732630561161, max timestamp=1732630561177Num uniques 1131; 2024-11-26T14:16:01,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-26T14:16:01,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1732630561161, max timestamp=1732630561177Num uniques 1131; 2024-11-26T14:16:01,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-26T14:16:01,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732630561170, max timestamp=1732630561177Num uniques 1131; 2024-11-26T14:16:01,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-26T14:16:01,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732630561211, max timestamp=1732630561217Num uniques 1140; 2024-11-26T14:16:01,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732630561211, max timestamp=1732630561217Num uniques 1140; 2024-11-26T14:16:01,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-26T14:16:01,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-26T14:16:01,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732630561211, max timestamp=1732630561217Num uniques 1040; 2024-11-26T14:16:01,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-26T14:16:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732630561240, max timestamp=1732630561256Num uniques 1049; 2024-11-26T14:16:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-26T14:16:01,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1732630561240, max timestamp=1732630561256Num uniques 1049; 2024-11-26T14:16:01,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-11-26T14:16:01,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561246, max timestamp=1732630561256Num uniques 1049; 2024-11-26T14:16:01,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=6870, suffix heap size=137064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=150 2024-11-26T14:16:01,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561285, max timestamp=1732630561300Num uniques 1158; 2024-11-26T14:16:01,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=6870, suffix heap size=148400,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=150 2024-11-26T14:16:01,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561285, max timestamp=1732630561300Num uniques 1158; 2024-11-26T14:16:01,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=6870, suffix heap size=148400,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=150 2024-11-26T14:16:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561285, max timestamp=1732630561300Num uniques 1009; 2024-11-26T14:16:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-26T14:16:01,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732630561342, max timestamp=1732630561350Num uniques 1009; 2024-11-26T14:16:01,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=9932, suffix heap size=132904,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=217 2024-11-26T14:16:01,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732630561342, max timestamp=1732630561350Num uniques 1009; 2024-11-26T14:16:01,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732630561342, max timestamp=1732630561350Num uniques 1009; 2024-11-26T14:16:01,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=9932, suffix heap size=132904,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=217 2024-11-26T14:16:01,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=9932, suffix heap size=132904,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=217 2024-11-26T14:16:01,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732630561384, max timestamp=1732630561400Num uniques 1076; 2024-11-26T14:16:01,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732630561384, max timestamp=1732630561400Num uniques 1076; 2024-11-26T14:16:01,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-26T14:16:01,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-26T14:16:01,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561384, max timestamp=1732630561400Num uniques 1076; 2024-11-26T14:16:01,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=6870, suffix heap size=139872,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=150 2024-11-26T14:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:01,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:01,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=385, cellSize=17630, totalHeapSize=40280, min timestamp=1732630561437, max timestamp=1732630561452Num uniques 1085; 2024-11-26T14:16:01,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=17630, suffix heap size=140808,new segment heap size=40280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=385 2024-11-26T14:16:01,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=385, cellSize=17630, totalHeapSize=40280, min timestamp=1732630561437, max timestamp=1732630561452Num uniques 1085; 2024-11-26T14:16:01,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=17630, suffix heap size=140808,new segment heap size=40280 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=385 2024-11-26T14:16:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1732630561437, max timestamp=1732630561452Num uniques 1009; 2024-11-26T14:16:01,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8470, suffix heap size=132904,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=185 2024-11-26T14:16:01,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732630561475, max timestamp=1732630561494Num uniques 1244; 2024-11-26T14:16:01,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=56974, new segment data size=13464, suffix heap size=157344,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1244, new segment cells count=294 2024-11-26T14:16:01,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732630561475, max timestamp=1732630561494Num uniques 1244; 2024-11-26T14:16:01,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=56974, new segment data size=13464, suffix heap size=157344,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1244, new segment cells count=294 2024-11-26T14:16:01,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732630561475, max timestamp=1732630561494Num uniques 1044; 2024-11-26T14:16:01,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-11-26T14:16:01,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/90ed99b67da046da9be83a082ec6dd26 2024-11-26T14:16:01,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732630561577, max timestamp=1732630561587Num uniques 1153; 2024-11-26T14:16:01,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732630561577, max timestamp=1732630561587Num uniques 1153; 2024-11-26T14:16:01,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-11-26T14:16:01,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-11-26T14:16:01,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732630561580, max timestamp=1732630561587Num uniques 1153; 2024-11-26T14:16:01,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-26T14:16:01,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/2413054938cb4d50bfcd518a4d27faf6 is 50, key is test_row_0/C:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:01,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742100_1276 (size=12151) 2024-11-26T14:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561623, max timestamp=1732630561643Num uniques 1062; 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=6870, suffix heap size=138416,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=150 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561623, max timestamp=1732630561643Num uniques 1062; 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=6870, suffix heap size=138416,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=150 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561623, max timestamp=1732630561643Num uniques 1012; 2024-11-26T14:16:01,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=6870, suffix heap size=133216,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=150 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1732630561666, max timestamp=1732630561689Num uniques 1009; 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=16986, suffix heap size=132904,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=371 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1732630561666, max timestamp=1732630561689Num uniques 1009; 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=16986, suffix heap size=132904,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=371 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=371, cellSize=16986, totalHeapSize=38824, min timestamp=1732630561666, max timestamp=1732630561689Num uniques 1009; 2024-11-26T14:16:01,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=16986, suffix heap size=132904,new segment heap size=38824 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=371 2024-11-26T14:16:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:01,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732630561718, max timestamp=1732630561780Num uniques 1230; 2024-11-26T14:16:01,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=10530, suffix heap size=155888,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=230 2024-11-26T14:16:01,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732630561718, max timestamp=1732630561780Num uniques 1230; 2024-11-26T14:16:01,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=10530, suffix heap size=155888,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=230 2024-11-26T14:16:01,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732630561718, max timestamp=1732630561780Num uniques 1230; 2024-11-26T14:16:01,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=56330, new segment data size=10530, suffix heap size=155888,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1230, new segment cells count=230 2024-11-26T14:16:01,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732630561834, max timestamp=1732630561840Num uniques 1089; 2024-11-26T14:16:01,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-26T14:16:01,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732630561834, max timestamp=1732630561840Num uniques 1089; 2024-11-26T14:16:01,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-26T14:16:01,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1732630561834, max timestamp=1732630561840Num uniques 1089; 2024-11-26T14:16:01,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=8654, suffix heap size=141224,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=189 2024-11-26T14:16:01,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9058, totalHeapSize=20832, min timestamp=1732630561884, max timestamp=1732630561904Num uniques 1048; 2024-11-26T14:16:01,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9058, totalHeapSize=20832, min timestamp=1732630561884, max timestamp=1732630561904Num uniques 1048; 2024-11-26T14:16:01,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47988, new segment data size=9058, suffix heap size=136960,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=198 2024-11-26T14:16:01,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47988, new segment data size=9058, suffix heap size=136960,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=198 2024-11-26T14:16:01,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1732630561884, max timestamp=1732630561904Num uniques 1048; 2024-11-26T14:16:01,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=9068, suffix heap size=136960,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=198 2024-11-26T14:16:01,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:01,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:01,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:01,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=157, cellSize=7185, totalHeapSize=16568, min timestamp=1732630561945, max timestamp=1732630561966Num uniques 1057; 2024-11-26T14:16:01,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=7185, suffix heap size=137896,new segment heap size=16568 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=157 2024-11-26T14:16:01,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561945, max timestamp=1732630561966Num uniques 1057; 2024-11-26T14:16:01,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=6870, suffix heap size=137896,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=150 2024-11-26T14:16:01,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630561945, max timestamp=1732630561966Num uniques 1057; 2024-11-26T14:16:01,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=6870, suffix heap size=137896,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=150 2024-11-26T14:16:02,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732630562004, max timestamp=1732630562014Num uniques 1016; 2024-11-26T14:16:02,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46526, new segment data size=9886, suffix heap size=133632,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1016, new segment cells count=216 2024-11-26T14:16:02,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732630562004, max timestamp=1732630562014Num uniques 1009; 2024-11-26T14:16:02,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=9886, suffix heap size=132904,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=216 2024-11-26T14:16:02,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732630562004, max timestamp=1732630562014Num uniques 1009; 2024-11-26T14:16:02,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=9886, suffix heap size=132904,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=216 2024-11-26T14:16:02,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/2413054938cb4d50bfcd518a4d27faf6 2024-11-26T14:16:02,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732630562032, max timestamp=1732630562049Num uniques 1075; 2024-11-26T14:16:02,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732630562032, max timestamp=1732630562049Num uniques 1075; 2024-11-26T14:16:02,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-26T14:16:02,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-26T14:16:02,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1732630562044, max timestamp=1732630562049Num uniques 1075; 2024-11-26T14:16:02,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=8010, suffix heap size=139768,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-11-26T14:16:02,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/2569bd7acc3e4b378a103b863e8c2a98 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98 2024-11-26T14:16:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98, entries=200, sequenceid=229, filesize=14.2 K 2024-11-26T14:16:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/90ed99b67da046da9be83a082ec6dd26 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26 2024-11-26T14:16:02,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T14:16:02,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/2413054938cb4d50bfcd518a4d27faf6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6 2024-11-26T14:16:02,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T14:16:02,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=78.27 KB/80150 for 66833f8312ba730f0c055c76a8c04f74 in 1430ms, sequenceid=229, compaction requested=true 2024-11-26T14:16:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:02,111 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:02,111 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:02,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=91.69 KB heapSize=254.19 KB 2024-11-26T14:16:02,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:02,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:02,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:02,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:02,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,115 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:02,115 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:02,115 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,115 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f7005ea15d4d4cc19bbd032d9cff0fe9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.0 K 2024-11-26T14:16:02,115 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:02,115 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:02,115 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,115 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e118470bc6ae478bb505f4dd1ad689bc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=38.4 K 2024-11-26T14:16:02,116 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting e118470bc6ae478bb505f4dd1ad689bc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558605 2024-11-26T14:16:02,117 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f7005ea15d4d4cc19bbd032d9cff0fe9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558605 2024-11-26T14:16:02,118 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 163a30543de545fe89b29dbaee440893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732630559275 2024-11-26T14:16:02,118 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f907fa74c55f46faa0cb6c05238d2994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732630559275 2024-11-26T14:16:02,119 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2569bd7acc3e4b378a103b863e8c2a98, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:02,119 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ed99b67da046da9be83a082ec6dd26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:02,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/934a920c3fb0412c9aa5312f06452e8e is 50, key is test_row_0/A:col10/1732630562111/Put/seqid=0 2024-11-26T14:16:02,135 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:02,135 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/9f47a86040ac4791887c298f2dc09b19 is 50, key is test_row_0/A:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:02,142 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:02,143 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/402c1d3f27b249e7978cbf2462c96882 is 50, key is test_row_0/B:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630622155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630622157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630622159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630622160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630622160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742102_1278 (size=12697) 2024-11-26T14:16:02,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742103_1279 (size=12697) 2024-11-26T14:16:02,204 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/402c1d3f27b249e7978cbf2462c96882 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/402c1d3f27b249e7978cbf2462c96882 2024-11-26T14:16:02,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742101_1277 (size=14741) 2024-11-26T14:16:02,220 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 402c1d3f27b249e7978cbf2462c96882(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:02,221 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:02,221 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630562111; duration=0sec 2024-11-26T14:16:02,221 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:02,221 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:02,221 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:02,225 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:02,225 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:02,225 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,225 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce01047b65604de4b19e0e2f91c3ee60, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.0 K 2024-11-26T14:16:02,225 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ce01047b65604de4b19e0e2f91c3ee60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732630558605 2024-11-26T14:16:02,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b905ce7fd1784b9e8362ecddd6d3fa7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732630559275 2024-11-26T14:16:02,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2413054938cb4d50bfcd518a4d27faf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:02,233 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:02,233 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/a7dc9502041f416796a66ea21398f96a is 50, key is test_row_0/C:col10/1732630560673/Put/seqid=0 2024-11-26T14:16:02,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630622274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630622274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630622274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630622274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742104_1280 (size=12697) 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630622274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:02,284 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/a7dc9502041f416796a66ea21398f96a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/a7dc9502041f416796a66ea21398f96a 2024-11-26T14:16:02,315 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into a7dc9502041f416796a66ea21398f96a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:02,315 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:02,315 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630562111; duration=0sec 2024-11-26T14:16:02,315 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:02,315 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:02,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630622483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630622483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630622484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630622484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630622484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:02,591 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/9f47a86040ac4791887c298f2dc09b19 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/9f47a86040ac4791887c298f2dc09b19 2024-11-26T14:16:02,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/934a920c3fb0412c9aa5312f06452e8e 2024-11-26T14:16:02,613 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into 9f47a86040ac4791887c298f2dc09b19(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:02,613 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:02,613 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630562110; duration=0sec 2024-11-26T14:16:02,613 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:02,613 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:02,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/a4c24bef7a474907b4f2c50afc7d8c2c is 50, key is test_row_0/B:col10/1732630562111/Put/seqid=0 2024-11-26T14:16:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742105_1281 (size=12301) 2024-11-26T14:16:02,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/a4c24bef7a474907b4f2c50afc7d8c2c 2024-11-26T14:16:02,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f2b366b5a2c44963b9812137d9405ebc is 50, key is test_row_0/C:col10/1732630562111/Put/seqid=0 2024-11-26T14:16:02,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742106_1282 (size=12301) 2024-11-26T14:16:02,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=674 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f2b366b5a2c44963b9812137d9405ebc 2024-11-26T14:16:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/934a920c3fb0412c9aa5312f06452e8e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e 2024-11-26T14:16:02,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e, entries=200, sequenceid=674, filesize=14.4 K 2024-11-26T14:16:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/a4c24bef7a474907b4f2c50afc7d8c2c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c 2024-11-26T14:16:02,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c, entries=150, sequenceid=674, filesize=12.0 K 2024-11-26T14:16:02,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f2b366b5a2c44963b9812137d9405ebc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc 2024-11-26T14:16:02,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc, entries=150, sequenceid=674, filesize=12.0 K 2024-11-26T14:16:02,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~293.98 KB/301040, currentSize=80.51 KB/82440 for 66833f8312ba730f0c055c76a8c04f74 in 591ms, sequenceid=674, compaction requested=false 2024-11-26T14:16:02,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:02,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T14:16:02,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:02,736 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fc10fa60ef79470b86d40d0a21209677 is 50, key is test_row_0/A:col10/1732630562151/Put/seqid=0 2024-11-26T14:16:02,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742107_1283 (size=12301) 2024-11-26T14:16:02,787 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fc10fa60ef79470b86d40d0a21209677 2024-11-26T14:16:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:02,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:02,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/03ef2b01f4404dcdb28bdc738bbbeac9 is 50, key is test_row_0/B:col10/1732630562151/Put/seqid=0 2024-11-26T14:16:02,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630622816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630622824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630622825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630622826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742108_1284 (size=12301) 2024-11-26T14:16:02,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630622830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,852 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/03ef2b01f4404dcdb28bdc738bbbeac9 2024-11-26T14:16:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/72e94b7dd74140199d00460e5a4e5135 is 50, key is test_row_0/C:col10/1732630562151/Put/seqid=0 2024-11-26T14:16:02,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630622936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742109_1285 (size=12301) 2024-11-26T14:16:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630622943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,952 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/72e94b7dd74140199d00460e5a4e5135 2024-11-26T14:16:02,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630622954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630622954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:02,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630622954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fc10fa60ef79470b86d40d0a21209677 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677 2024-11-26T14:16:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677, entries=150, sequenceid=692, filesize=12.0 K 2024-11-26T14:16:03,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/03ef2b01f4404dcdb28bdc738bbbeac9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9 2024-11-26T14:16:03,032 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9, entries=150, sequenceid=692, filesize=12.0 K 2024-11-26T14:16:03,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/72e94b7dd74140199d00460e5a4e5135 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135 2024-11-26T14:16:03,042 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135, entries=150, sequenceid=692, filesize=12.0 K 2024-11-26T14:16:03,048 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 66833f8312ba730f0c055c76a8c04f74 in 312ms, sequenceid=692, compaction requested=true 2024-11-26T14:16:03,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:03,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T14:16:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T14:16:03,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T14:16:03,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-11-26T14:16:03,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.9620 sec 2024-11-26T14:16:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:03,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:03,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/62b28ca1e9f64987aa3094f840154852 is 50, key is test_row_0/A:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742110_1286 (size=14741) 2024-11-26T14:16:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630623171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630623170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/62b28ca1e9f64987aa3094f840154852 2024-11-26T14:16:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630623176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630623178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630623184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/74b19bb6cf94461da4f82d584c82358d is 50, key is test_row_0/B:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742111_1287 (size=12301) 2024-11-26T14:16:03,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/74b19bb6cf94461da4f82d584c82358d 2024-11-26T14:16:03,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b873cefa8b324b2a877996250ec7068f is 50, key is test_row_0/C:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742112_1288 (size=12301) 2024-11-26T14:16:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630623282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630623283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T14:16:03,293 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:03,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-26T14:16:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T14:16:03,296 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:03,297 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:03,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630623293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630623293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630623293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T14:16:03,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T14:16:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630623493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630623493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630623503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630623504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630623504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T14:16:03,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T14:16:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:03,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b873cefa8b324b2a877996250ec7068f 2024-11-26T14:16:03,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/62b28ca1e9f64987aa3094f840154852 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852 2024-11-26T14:16:03,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852, entries=200, sequenceid=711, filesize=14.4 K 2024-11-26T14:16:03,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/74b19bb6cf94461da4f82d584c82358d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d 2024-11-26T14:16:03,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d, entries=150, sequenceid=711, filesize=12.0 K 2024-11-26T14:16:03,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b873cefa8b324b2a877996250ec7068f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f 2024-11-26T14:16:03,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f, entries=150, sequenceid=711, filesize=12.0 K 2024-11-26T14:16:03,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 66833f8312ba730f0c055c76a8c04f74 in 572ms, sequenceid=711, compaction requested=true 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:03,721 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:03,721 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:03,731 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:16:03,731 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:16:03,731 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:03,731 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:03,731 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,731 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,731 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/9f47a86040ac4791887c298f2dc09b19, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=53.2 K 2024-11-26T14:16:03,731 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/402c1d3f27b249e7978cbf2462c96882, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=48.4 K 2024-11-26T14:16:03,731 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f47a86040ac4791887c298f2dc09b19, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:03,732 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 402c1d3f27b249e7978cbf2462c96882, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:03,733 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934a920c3fb0412c9aa5312f06452e8e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732630562075 2024-11-26T14:16:03,736 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c24bef7a474907b4f2c50afc7d8c2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732630562075 2024-11-26T14:16:03,736 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc10fa60ef79470b86d40d0a21209677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732630562132 2024-11-26T14:16:03,740 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ef2b01f4404dcdb28bdc738bbbeac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732630562132 2024-11-26T14:16:03,740 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b28ca1e9f64987aa3094f840154852, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562818 2024-11-26T14:16:03,741 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b19bb6cf94461da4f82d584c82358d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562821 2024-11-26T14:16:03,763 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:03,764 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/69d595fa967f41928089ea940a675ebe is 50, key is test_row_0/A:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,775 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:03,775 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f02b6945a5bf494cb6c33ce81c989a07 is 50, key is test_row_0/B:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742113_1289 (size=12983) 2024-11-26T14:16:03,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T14:16:03,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,792 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:16:03,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742114_1290 (size=12983) 2024-11-26T14:16:03,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:03,808 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/69d595fa967f41928089ea940a675ebe as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/69d595fa967f41928089ea940a675ebe 2024-11-26T14:16:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/73c5f5c419f547dd9cb08aad590f4a83 is 50, key is test_row_0/A:col10/1732630563176/Put/seqid=0 2024-11-26T14:16:03,824 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f02b6945a5bf494cb6c33ce81c989a07 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f02b6945a5bf494cb6c33ce81c989a07 2024-11-26T14:16:03,827 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into 69d595fa967f41928089ea940a675ebe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:03,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:03,827 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=12, startTime=1732630563721; duration=0sec 2024-11-26T14:16:03,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:03,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:03,827 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:16:03,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742115_1291 (size=12301) 2024-11-26T14:16:03,831 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:16:03,831 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:03,831 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:03,831 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/a7dc9502041f416796a66ea21398f96a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=48.4 K 2024-11-26T14:16:03,832 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/73c5f5c419f547dd9cb08aad590f4a83 2024-11-26T14:16:03,832 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7dc9502041f416796a66ea21398f96a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732630560021 2024-11-26T14:16:03,834 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b366b5a2c44963b9812137d9405ebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=674, earliestPutTs=1732630562075 2024-11-26T14:16:03,836 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e94b7dd74140199d00460e5a4e5135, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=692, earliestPutTs=1732630562132 2024-11-26T14:16:03,836 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b873cefa8b324b2a877996250ec7068f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562821 2024-11-26T14:16:03,841 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into f02b6945a5bf494cb6c33ce81c989a07(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:03,841 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:03,841 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=12, startTime=1732630563721; duration=0sec 2024-11-26T14:16:03,841 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:03,841 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/54538094eab04a3784323f59f43ee47a is 50, key is test_row_0/B:col10/1732630563176/Put/seqid=0 2024-11-26T14:16:03,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630623855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630623857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630623857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630623858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,866 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:03,866 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/95a5b929f3574837a62d38223ea42b68 is 50, key is test_row_0/C:col10/1732630563147/Put/seqid=0 2024-11-26T14:16:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630623868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742116_1292 (size=12301) 2024-11-26T14:16:03,901 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/54538094eab04a3784323f59f43ee47a 2024-11-26T14:16:03,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742117_1293 (size=12983) 2024-11-26T14:16:03,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fc09cd4246cd4dd0904eb9aee1738769 is 50, key is test_row_0/C:col10/1732630563176/Put/seqid=0 2024-11-26T14:16:03,921 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/95a5b929f3574837a62d38223ea42b68 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95a5b929f3574837a62d38223ea42b68 2024-11-26T14:16:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T14:16:03,927 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into 95a5b929f3574837a62d38223ea42b68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:03,927 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:03,927 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=12, startTime=1732630563721; duration=0sec 2024-11-26T14:16:03,927 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:03,928 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742118_1294 (size=12301) 2024-11-26T14:16:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630623973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630623973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630623973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630623973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630623983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630624183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630624183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630624183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630624184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630624197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,342 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fc09cd4246cd4dd0904eb9aee1738769 2024-11-26T14:16:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/73c5f5c419f547dd9cb08aad590f4a83 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83 2024-11-26T14:16:04,362 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83, entries=150, sequenceid=724, filesize=12.0 K 2024-11-26T14:16:04,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/54538094eab04a3784323f59f43ee47a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a 2024-11-26T14:16:04,371 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a, entries=150, sequenceid=724, filesize=12.0 K 2024-11-26T14:16:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fc09cd4246cd4dd0904eb9aee1738769 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769 2024-11-26T14:16:04,378 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769, entries=150, sequenceid=724, filesize=12.0 K 2024-11-26T14:16:04,380 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 66833f8312ba730f0c055c76a8c04f74 in 588ms, sequenceid=724, compaction requested=false 2024-11-26T14:16:04,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T14:16:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T14:16:04,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T14:16:04,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-11-26T14:16:04,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.0910 sec 2024-11-26T14:16:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T14:16:04,444 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:04,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-26T14:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:04,452 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:04,455 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:04,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:04,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-26T14:16:04,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732630563825, max timestamp=1732630564499Num uniques 859; 2024-11-26T14:16:04,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:16:04,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732630563825, max timestamp=1732630564499Num uniques 859; 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:04,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:04,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/df11122eb7534fc089abaa6c74885bca is 50, key is test_row_0/A:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:04,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742119_1295 (size=12301) 2024-11-26T14:16:04,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/df11122eb7534fc089abaa6c74885bca 2024-11-26T14:16:04,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f03fb5a6fb66431cb8071a5daa07859d is 50, key is test_row_0/B:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742120_1296 (size=12301) 2024-11-26T14:16:04,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f03fb5a6fb66431cb8071a5daa07859d 2024-11-26T14:16:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630624589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630624597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630624599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630624599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/9ac66af9dfc5426caeb0f8ddc138813f is 50, key is test_row_0/C:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630624601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742121_1297 (size=12301) 2024-11-26T14:16:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630624705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630624714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630624714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630624714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630624716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T14:16:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630624916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T14:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:04,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:04,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630624923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630624933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630624934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630624934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/9ac66af9dfc5426caeb0f8ddc138813f 2024-11-26T14:16:05,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/df11122eb7534fc089abaa6c74885bca as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca 2024-11-26T14:16:05,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca, entries=150, sequenceid=750, filesize=12.0 K 2024-11-26T14:16:05,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/f03fb5a6fb66431cb8071a5daa07859d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d 2024-11-26T14:16:05,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d, entries=150, sequenceid=750, filesize=12.0 K 2024-11-26T14:16:05,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/9ac66af9dfc5426caeb0f8ddc138813f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f 2024-11-26T14:16:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:05,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f, entries=150, sequenceid=750, filesize=12.0 K 2024-11-26T14:16:05,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~214.13 KB/219264, currentSize=107.34 KB/109920 for 66833f8312ba730f0c055c76a8c04f74 in 534ms, sequenceid=750, compaction requested=true 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:05,081 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:05,081 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:05,083 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:05,084 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:05,084 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f02b6945a5bf494cb6c33ce81c989a07, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.7 K 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f02b6945a5bf494cb6c33ce81c989a07, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562821 2024-11-26T14:16:05,084 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:05,084 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/69d595fa967f41928089ea940a675ebe, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.7 K 2024-11-26T14:16:05,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 54538094eab04a3784323f59f43ee47a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1732630563166 2024-11-26T14:16:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:05,084 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d595fa967f41928089ea940a675ebe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562821 2024-11-26T14:16:05,084 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:16:05,085 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f03fb5a6fb66431cb8071a5daa07859d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:05,085 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c5f5c419f547dd9cb08aad590f4a83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1732630563166 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:05,085 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting df11122eb7534fc089abaa6c74885bca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d5762d6828d144628b12c73b076853d3 is 50, key is test_row_0/A:col10/1732630564586/Put/seqid=0 2024-11-26T14:16:05,099 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:05,100 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/0ade4b5eddfa4b92bdcbc6b984803c2d is 50, key is test_row_0/A:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:05,101 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:05,102 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/d7ed522e845c40ac82c81ce8506b708a is 50, key is test_row_0/B:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:05,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742122_1298 (size=12301) 2024-11-26T14:16:05,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742123_1299 (size=13085) 2024-11-26T14:16:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742124_1300 (size=13085) 2024-11-26T14:16:05,124 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/0ade4b5eddfa4b92bdcbc6b984803c2d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/0ade4b5eddfa4b92bdcbc6b984803c2d 2024-11-26T14:16:05,133 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/d7ed522e845c40ac82c81ce8506b708a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d7ed522e845c40ac82c81ce8506b708a 2024-11-26T14:16:05,135 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into 0ade4b5eddfa4b92bdcbc6b984803c2d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:05,135 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:05,135 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630565081; duration=0sec 2024-11-26T14:16:05,136 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:05,136 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:05,136 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:05,137 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:05,137 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:05,137 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:05,137 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95a5b929f3574837a62d38223ea42b68, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.7 K 2024-11-26T14:16:05,138 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95a5b929f3574837a62d38223ea42b68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=711, earliestPutTs=1732630562821 2024-11-26T14:16:05,139 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc09cd4246cd4dd0904eb9aee1738769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1732630563166 2024-11-26T14:16:05,140 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac66af9dfc5426caeb0f8ddc138813f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:05,146 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into d7ed522e845c40ac82c81ce8506b708a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:05,146 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:05,147 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630565081; duration=0sec 2024-11-26T14:16:05,147 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:05,147 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:05,156 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:05,157 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/34421732fdb049f58ed09e1d0f71bf67 is 50, key is test_row_0/C:col10/1732630564501/Put/seqid=0 2024-11-26T14:16:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742125_1301 (size=13085) 2024-11-26T14:16:05,213 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/34421732fdb049f58ed09e1d0f71bf67 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/34421732fdb049f58ed09e1d0f71bf67 2024-11-26T14:16:05,228 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into 34421732fdb049f58ed09e1d0f71bf67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:05,228 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:05,228 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630565081; duration=0sec 2024-11-26T14:16:05,228 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:05,228 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:05,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630625272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630625277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630625280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630625280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630625285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630625393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630625394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630625404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630625404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630625405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,512 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=769 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d5762d6828d144628b12c73b076853d3 2024-11-26T14:16:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/0e510804880647b39ceaa2b22bdef54d is 50, key is test_row_0/B:col10/1732630564586/Put/seqid=0 2024-11-26T14:16:05,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742126_1302 (size=12301) 2024-11-26T14:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:05,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630625603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630625604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630625613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630625613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630625616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630625913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630625913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630625923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630625923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630625933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:05,972 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=769 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/0e510804880647b39ceaa2b22bdef54d 2024-11-26T14:16:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/e517d68f0975409a96e9efabf0837089 is 50, key is test_row_0/C:col10/1732630564586/Put/seqid=0 2024-11-26T14:16:05,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742127_1303 (size=12301) 2024-11-26T14:16:05,984 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=769 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/e517d68f0975409a96e9efabf0837089 2024-11-26T14:16:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/d5762d6828d144628b12c73b076853d3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3 2024-11-26T14:16:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3, entries=150, sequenceid=769, filesize=12.0 K 2024-11-26T14:16:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/0e510804880647b39ceaa2b22bdef54d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d 2024-11-26T14:16:05,999 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d, entries=150, sequenceid=769, filesize=12.0 K 2024-11-26T14:16:06,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/e517d68f0975409a96e9efabf0837089 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089 2024-11-26T14:16:06,007 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089, entries=150, sequenceid=769, filesize=12.0 K 2024-11-26T14:16:06,008 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 66833f8312ba730f0c055c76a8c04f74 in 924ms, sequenceid=769, compaction requested=false 2024-11-26T14:16:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T14:16:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T14:16:06,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T14:16:06,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-26T14:16:06,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5840 sec 2024-11-26T14:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:06,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:16:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:06,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:06,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/72bae0da597040058311c6bc4e696f95 is 50, key is test_row_0/A:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742128_1304 (size=12301) 2024-11-26T14:16:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630626452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630626453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630626454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630626455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630626456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630626563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630626563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630626563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630626563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630626563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T14:16:06,593 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:06,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T14:16:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:06,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:06,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:06,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:06,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T14:16:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:06,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630626773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630626773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630626773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630626774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630626774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:06,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:06,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/72bae0da597040058311c6bc4e696f95 2024-11-26T14:16:06,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/728d4ba6061b44e6b18109e8f53b5671 is 50, key is test_row_0/B:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:06,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742129_1305 (size=12301) 2024-11-26T14:16:06,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/728d4ba6061b44e6b18109e8f53b5671 2024-11-26T14:16:06,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/8b66617a1f64477c96ac92615b7bd24d is 50, key is test_row_0/C:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:06,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742130_1306 (size=12301) 2024-11-26T14:16:06,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T14:16:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:06,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:07,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T14:16:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T14:16:07,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:07,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:07,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/8b66617a1f64477c96ac92615b7bd24d 2024-11-26T14:16:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/72bae0da597040058311c6bc4e696f95 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95 2024-11-26T14:16:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95, entries=150, sequenceid=788, filesize=12.0 K 2024-11-26T14:16:07,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/728d4ba6061b44e6b18109e8f53b5671 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671 2024-11-26T14:16:07,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671, entries=150, sequenceid=788, filesize=12.0 K 2024-11-26T14:16:07,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/8b66617a1f64477c96ac92615b7bd24d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d 2024-11-26T14:16:07,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d, entries=150, sequenceid=788, filesize=12.0 K 2024-11-26T14:16:07,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 66833f8312ba730f0c055c76a8c04f74 in 879ms, sequenceid=788, compaction requested=true 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:07,307 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:07,307 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:07,308 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,308 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,308 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/0ade4b5eddfa4b92bdcbc6b984803c2d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.8 K 2024-11-26T14:16:07,308 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d7ed522e845c40ac82c81ce8506b708a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.8 K 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ed522e845c40ac82c81ce8506b708a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:07,308 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ade4b5eddfa4b92bdcbc6b984803c2d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:07,309 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e510804880647b39ceaa2b22bdef54d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=769, earliestPutTs=1732630564586 2024-11-26T14:16:07,309 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5762d6828d144628b12c73b076853d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=769, earliestPutTs=1732630564586 2024-11-26T14:16:07,309 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 728d4ba6061b44e6b18109e8f53b5671, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:07,309 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bae0da597040058311c6bc4e696f95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:07,318 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:07,319 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/a5a2014514b54208b5137a705bc8684f is 50, key is test_row_0/A:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:07,324 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:07,324 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/8c5c24e0ef444a7783554729175fcefe is 50, key is test_row_0/B:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:07,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742131_1307 (size=13187) 2024-11-26T14:16:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742132_1308 (size=13187) 2024-11-26T14:16:07,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T14:16:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,362 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:07,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:07,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/57fbcdbedcac4137b946bc3d425fd885 is 50, key is test_row_0/A:col10/1732630566453/Put/seqid=0 2024-11-26T14:16:07,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742133_1309 (size=12301) 2024-11-26T14:16:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:07,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:07,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630627608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630627609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630627610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630627612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630627613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630627723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630627723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630627723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630627723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630627723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,734 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/a5a2014514b54208b5137a705bc8684f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/a5a2014514b54208b5137a705bc8684f 2024-11-26T14:16:07,734 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/8c5c24e0ef444a7783554729175fcefe as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8c5c24e0ef444a7783554729175fcefe 2024-11-26T14:16:07,744 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 8c5c24e0ef444a7783554729175fcefe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:07,744 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:07,744 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630567307; duration=0sec 2024-11-26T14:16:07,744 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:07,744 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:07,744 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:07,745 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into a5a2014514b54208b5137a705bc8684f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:07,745 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:07,746 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630567307; duration=0sec 2024-11-26T14:16:07,746 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:07,746 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:07,746 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:07,746 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:07,746 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:07,746 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/34421732fdb049f58ed09e1d0f71bf67, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.8 K 2024-11-26T14:16:07,747 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 34421732fdb049f58ed09e1d0f71bf67, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1732630564499 2024-11-26T14:16:07,747 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e517d68f0975409a96e9efabf0837089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=769, earliestPutTs=1732630564586 2024-11-26T14:16:07,748 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b66617a1f64477c96ac92615b7bd24d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:07,770 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/57fbcdbedcac4137b946bc3d425fd885 2024-11-26T14:16:07,771 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:07,771 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/59c8bd3262cb4baebc4cc60b5c309ad1 is 50, key is test_row_0/C:col10/1732630566426/Put/seqid=0 2024-11-26T14:16:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/11f4f1edd05e4593b2d9a29f451936e7 is 50, key is test_row_0/B:col10/1732630566453/Put/seqid=0 2024-11-26T14:16:07,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742134_1310 (size=13187) 2024-11-26T14:16:07,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742135_1311 (size=12301) 2024-11-26T14:16:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630627932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630627933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630627943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,195 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/11f4f1edd05e4593b2d9a29f451936e7 2024-11-26T14:16:08,206 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/59c8bd3262cb4baebc4cc60b5c309ad1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/59c8bd3262cb4baebc4cc60b5c309ad1 2024-11-26T14:16:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f81f526844524fcc86db465539d5f439 is 50, key is test_row_0/C:col10/1732630566453/Put/seqid=0 2024-11-26T14:16:08,223 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into 59c8bd3262cb4baebc4cc60b5c309ad1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:08,223 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:08,223 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630567307; duration=0sec 2024-11-26T14:16:08,223 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:08,223 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742136_1312 (size=12301) 2024-11-26T14:16:08,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630628244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630628244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,254 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=804 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f81f526844524fcc86db465539d5f439 2024-11-26T14:16:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630628253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630628253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/57fbcdbedcac4137b946bc3d425fd885 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885 2024-11-26T14:16:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630628253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,264 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885, entries=150, sequenceid=804, filesize=12.0 K 2024-11-26T14:16:08,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/11f4f1edd05e4593b2d9a29f451936e7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7 2024-11-26T14:16:08,274 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7, entries=150, sequenceid=804, filesize=12.0 K 2024-11-26T14:16:08,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/f81f526844524fcc86db465539d5f439 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439 2024-11-26T14:16:08,284 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439, entries=150, sequenceid=804, filesize=12.0 K 2024-11-26T14:16:08,285 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 66833f8312ba730f0c055c76a8c04f74 in 924ms, sequenceid=804, compaction requested=false 2024-11-26T14:16:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T14:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T14:16:08,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T14:16:08,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-26T14:16:08,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6960 sec 2024-11-26T14:16:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T14:16:08,743 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:08,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T14:16:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:08,745 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:08,745 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:08,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:08,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:16:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:08,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fa054e992074466984c67cce6198cd99 is 50, key is test_row_0/A:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:08,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742137_1313 (size=14741) 2024-11-26T14:16:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630628775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630628777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630628779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630628780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630628789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:08,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:08,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630628893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630628893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630628893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630628894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630628902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630629103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630629103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630629103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630629103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630629113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fa054e992074466984c67cce6198cd99 2024-11-26T14:16:09,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3f30e76b5fc141a88a6ba3323d4b98ba is 50, key is test_row_0/B:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:09,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742138_1314 (size=12301) 2024-11-26T14:16:09,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630629412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630629413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630629413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630629413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630629423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3f30e76b5fc141a88a6ba3323d4b98ba 2024-11-26T14:16:09,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:09,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/4a847483346d4bb79bec94e064a48980 is 50, key is test_row_0/C:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:09,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742139_1315 (size=12301) 2024-11-26T14:16:09,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/4a847483346d4bb79bec94e064a48980 2024-11-26T14:16:09,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/fa054e992074466984c67cce6198cd99 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99 2024-11-26T14:16:09,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99, entries=200, sequenceid=824, filesize=14.4 K 2024-11-26T14:16:09,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3f30e76b5fc141a88a6ba3323d4b98ba as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba 2024-11-26T14:16:09,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba, entries=150, sequenceid=824, filesize=12.0 K 2024-11-26T14:16:09,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/4a847483346d4bb79bec94e064a48980 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980 2024-11-26T14:16:09,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980, entries=150, sequenceid=824, filesize=12.0 K 2024-11-26T14:16:09,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 66833f8312ba730f0c055c76a8c04f74 in 1009ms, sequenceid=824, compaction requested=true 2024-11-26T14:16:09,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:09,765 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:09,765 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:09,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:09,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:09,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:09,766 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:09,766 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:09,767 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:09,767 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:09,767 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,767 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,767 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8c5c24e0ef444a7783554729175fcefe, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.9 K 2024-11-26T14:16:09,767 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/a5a2014514b54208b5137a705bc8684f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=39.3 K 2024-11-26T14:16:09,767 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c5c24e0ef444a7783554729175fcefe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:09,768 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5a2014514b54208b5137a705bc8684f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:09,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f4f1edd05e4593b2d9a29f451936e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1732630566453 2024-11-26T14:16:09,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f30e76b5fc141a88a6ba3323d4b98ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:09,768 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57fbcdbedcac4137b946bc3d425fd885, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1732630566453 2024-11-26T14:16:09,769 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa054e992074466984c67cce6198cd99, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:09,778 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:09,778 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/1ff0eb4099504528b3c1d29f5b545aa3 is 50, key is test_row_0/B:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:09,784 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#270 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:09,785 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b0ef5daf33014df49255bf10a423a8fd is 50, key is test_row_0/A:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:09,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742140_1316 (size=13289) 2024-11-26T14:16:09,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742141_1317 (size=13289) 2024-11-26T14:16:09,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T14:16:09,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,820 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:09,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:09,830 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/1ff0eb4099504528b3c1d29f5b545aa3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/1ff0eb4099504528b3c1d29f5b545aa3 2024-11-26T14:16:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/407a7d778bc640f69ec4b098f99e9e83 is 50, key is test_row_0/A:col10/1732630568779/Put/seqid=0 2024-11-26T14:16:09,840 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 1ff0eb4099504528b3c1d29f5b545aa3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:09,840 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:09,840 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630569765; duration=0sec 2024-11-26T14:16:09,840 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:09,840 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:09,840 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:09,841 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:09,842 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:09,842 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:09,842 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/59c8bd3262cb4baebc4cc60b5c309ad1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=36.9 K 2024-11-26T14:16:09,842 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c8bd3262cb4baebc4cc60b5c309ad1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732630565271 2024-11-26T14:16:09,843 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f81f526844524fcc86db465539d5f439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=804, earliestPutTs=1732630566453 2024-11-26T14:16:09,843 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a847483346d4bb79bec94e064a48980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742142_1318 (size=12301) 2024-11-26T14:16:09,855 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:09,857 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/369bb7e2a2314cc4957a60d7107daac6 is 50, key is test_row_0/C:col10/1732630567610/Put/seqid=0 2024-11-26T14:16:09,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742143_1319 (size=13289) 2024-11-26T14:16:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. as already flushing 2024-11-26T14:16:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630629938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630629939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630629944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630629944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630629944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,216 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b0ef5daf33014df49255bf10a423a8fd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b0ef5daf33014df49255bf10a423a8fd 2024-11-26T14:16:10,220 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into b0ef5daf33014df49255bf10a423a8fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:10,220 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:10,220 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630569764; duration=0sec 2024-11-26T14:16:10,220 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:10,220 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:10,249 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=840 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/407a7d778bc640f69ec4b098f99e9e83 2024-11-26T14:16:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3e46d3e8c5f642df8efcc0e217058aee is 50, key is test_row_0/B:col10/1732630568779/Put/seqid=0 2024-11-26T14:16:10,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742144_1320 (size=12301) 2024-11-26T14:16:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630630263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630630263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630630263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630630263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630630263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,270 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/369bb7e2a2314cc4957a60d7107daac6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/369bb7e2a2314cc4957a60d7107daac6 2024-11-26T14:16:10,275 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into 369bb7e2a2314cc4957a60d7107daac6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:10,275 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:10,275 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630569766; duration=0sec 2024-11-26T14:16:10,275 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:10,275 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630630573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630630574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630630574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630630574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630630574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:10,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:10,659 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=840 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3e46d3e8c5f642df8efcc0e217058aee 2024-11-26T14:16:10,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/510e20d952ae44058d92c18739fbc76b is 50, key is test_row_0/C:col10/1732630568779/Put/seqid=0 2024-11-26T14:16:10,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742145_1321 (size=12301) 2024-11-26T14:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:11,072 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=840 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/510e20d952ae44058d92c18739fbc76b 2024-11-26T14:16:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/407a7d778bc640f69ec4b098f99e9e83 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83 2024-11-26T14:16:11,082 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83, entries=150, sequenceid=840, filesize=12.0 K 2024-11-26T14:16:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/3e46d3e8c5f642df8efcc0e217058aee as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee 2024-11-26T14:16:11,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630631083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630631083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630631083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:11,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630631083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:11,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630631083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:11,090 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee, entries=150, sequenceid=840, filesize=12.0 K 2024-11-26T14:16:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/510e20d952ae44058d92c18739fbc76b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b 2024-11-26T14:16:11,097 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b, entries=150, sequenceid=840, filesize=12.0 K 2024-11-26T14:16:11,098 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 66833f8312ba730f0c055c76a8c04f74 in 1278ms, sequenceid=840, compaction requested=false 2024-11-26T14:16:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T14:16:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T14:16:11,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T14:16:11,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3670 sec 2024-11-26T14:16:11,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3730 sec 2024-11-26T14:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:12,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:12,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:12,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b7d2cc9c500b474c8354f0064d63a0db is 50, key is test_row_0/A:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742146_1322 (size=12301) 2024-11-26T14:16:12,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630632105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630632106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630632107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630632111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630632112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630632223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630632223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630632223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630632223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630632223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630632433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=863 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b7d2cc9c500b474c8354f0064d63a0db 2024-11-26T14:16:12,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/aa2b6ae9d7504286a0bce04d9eb3024c is 50, key is test_row_0/B:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:12,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742147_1323 (size=12301) 2024-11-26T14:16:12,645 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-11-26T14:16:12,645 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:12,645 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,645 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:12,646 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-26T14:16:12,646 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:12,646 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,646 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:12,647 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-26T14:16:12,647 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:12,647 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,647 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:12,649 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-26T14:16:12,649 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:12,649 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,649 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:12,651 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-26T14:16:12,651 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:12,651 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,652 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:12,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:12,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630632743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630632743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630632743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630632743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630632744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T14:16:12,914 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:12,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=863 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/aa2b6ae9d7504286a0bce04d9eb3024c 2024-11-26T14:16:12,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fe4acd3f192c436f951bb92d0c3d982d is 50, key is test_row_0/C:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:12,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742148_1324 (size=12301) 2024-11-26T14:16:13,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56638 deadline: 1732630633253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56606 deadline: 1732630633254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56612 deadline: 1732630633254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56596 deadline: 1732630633254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56634 deadline: 1732630633254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:16:13,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=863 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fe4acd3f192c436f951bb92d0c3d982d 2024-11-26T14:16:13,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/b7d2cc9c500b474c8354f0064d63a0db as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db 2024-11-26T14:16:13,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db, entries=150, sequenceid=863, filesize=12.0 K 2024-11-26T14:16:13,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/aa2b6ae9d7504286a0bce04d9eb3024c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c 2024-11-26T14:16:13,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c, entries=150, sequenceid=863, filesize=12.0 K 2024-11-26T14:16:13,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/fe4acd3f192c436f951bb92d0c3d982d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d 2024-11-26T14:16:13,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d, entries=150, sequenceid=863, filesize=12.0 K 2024-11-26T14:16:13,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 66833f8312ba730f0c055c76a8c04f74 in 1267ms, sequenceid=863, compaction requested=true 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:13,362 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66833f8312ba730f0c055c76a8c04f74:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:13,362 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/B is initiating minor compaction (all files) 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/A is initiating minor compaction (all files) 2024-11-26T14:16:13,363 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/B in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:13,363 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/A in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:13,363 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/1ff0eb4099504528b3c1d29f5b545aa3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=37.0 K 2024-11-26T14:16:13,363 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b0ef5daf33014df49255bf10a423a8fd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=37.0 K 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff0eb4099504528b3c1d29f5b545aa3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:13,363 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ef5daf33014df49255bf10a423a8fd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:13,364 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 407a7d778bc640f69ec4b098f99e9e83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=840, earliestPutTs=1732630568777 2024-11-26T14:16:13,364 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e46d3e8c5f642df8efcc0e217058aee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=840, earliestPutTs=1732630568777 2024-11-26T14:16:13,364 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d2cc9c500b474c8354f0064d63a0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=863, earliestPutTs=1732630569933 2024-11-26T14:16:13,364 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2b6ae9d7504286a0bce04d9eb3024c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=863, earliestPutTs=1732630569933 2024-11-26T14:16:13,369 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#B#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:13,369 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:13,369 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/c1959fa1add3431babe3742973032409 is 50, key is test_row_0/A:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:13,370 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/850d6450281d4cabaa0445939e87ec96 is 50, key is test_row_0/B:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742150_1326 (size=13391) 2024-11-26T14:16:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742149_1325 (size=13391) 2024-11-26T14:16:13,778 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/c1959fa1add3431babe3742973032409 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/c1959fa1add3431babe3742973032409 2024-11-26T14:16:13,778 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/850d6450281d4cabaa0445939e87ec96 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/850d6450281d4cabaa0445939e87ec96 2024-11-26T14:16:13,783 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/A of 66833f8312ba730f0c055c76a8c04f74 into c1959fa1add3431babe3742973032409(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:13,783 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:13,783 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/A, priority=13, startTime=1732630573362; duration=0sec 2024-11-26T14:16:13,783 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:13,783 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:A 2024-11-26T14:16:13,783 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:13,784 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:13,784 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 66833f8312ba730f0c055c76a8c04f74/C is initiating minor compaction (all files) 2024-11-26T14:16:13,784 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66833f8312ba730f0c055c76a8c04f74/C in TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:13,784 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/369bb7e2a2314cc4957a60d7107daac6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp, totalSize=37.0 K 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 369bb7e2a2314cc4957a60d7107daac6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=824, earliestPutTs=1732630567608 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510e20d952ae44058d92c18739fbc76b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=840, earliestPutTs=1732630568777 2024-11-26T14:16:13,785 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/B of 66833f8312ba730f0c055c76a8c04f74 into 850d6450281d4cabaa0445939e87ec96(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:13,785 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/B, priority=13, startTime=1732630573362; duration=0sec 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4acd3f192c436f951bb92d0c3d982d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=863, earliestPutTs=1732630569933 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:13,785 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:B 2024-11-26T14:16:13,792 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66833f8312ba730f0c055c76a8c04f74#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:13,793 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b7e69c03ecaf4069ae3107e114a63885 is 50, key is test_row_0/C:col10/1732630572096/Put/seqid=0 2024-11-26T14:16:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742151_1327 (size=13391) 2024-11-26T14:16:13,802 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/b7e69c03ecaf4069ae3107e114a63885 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b7e69c03ecaf4069ae3107e114a63885 2024-11-26T14:16:13,807 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66833f8312ba730f0c055c76a8c04f74/C of 66833f8312ba730f0c055c76a8c04f74 into b7e69c03ecaf4069ae3107e114a63885(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:13,807 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:13,807 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., storeName=66833f8312ba730f0c055c76a8c04f74/C, priority=13, startTime=1732630573362; duration=0sec 2024-11-26T14:16:13,808 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:13,808 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66833f8312ba730f0c055c76a8c04f74:C 2024-11-26T14:16:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:14,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:16:14,270 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-26T14:16:14,270 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-26T14:16:14,270 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:14,270 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:14,270 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-26T14:16:14,270 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:14,270 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:14,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,270 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:14,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:14,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:14,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/5bb094bac057446b956a9f73a7d4cd0f is 50, key is test_row_0/A:col10/1732630574265/Put/seqid=0 2024-11-26T14:16:14,275 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-26T14:16:14,275 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:14,275 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,275 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-26T14:16:14,275 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:14,275 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,275 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,276 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 217 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 160 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2065 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2079 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2066 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2071 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2065 2024-11-26T14:16:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:16:14,276 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:16:14,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:16:14,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:14,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:14,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:16:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:14,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742152_1328 (size=12301) 2024-11-26T14:16:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:14,281 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630574281"}]},"ts":"1732630574281"} 2024-11-26T14:16:14,282 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:16:14,282 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:16:14,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:16:14,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, UNASSIGN}] 2024-11-26T14:16:14,285 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, UNASSIGN 2024-11-26T14:16:14,285 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=66833f8312ba730f0c055c76a8c04f74, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:14,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, UNASSIGN because future has completed 2024-11-26T14:16:14,287 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:16:14,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:14,439 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(122): Close 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:14,439 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:16:14,440 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1722): Closing 66833f8312ba730f0c055c76a8c04f74, disabling compactions & flushes 2024-11-26T14:16:14,440 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:14,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=881 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/5bb094bac057446b956a9f73a7d4cd0f 2024-11-26T14:16:14,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/bbfe5311d8c84149af544ad7585c25b4 is 50, key is test_row_0/B:col10/1732630574265/Put/seqid=0 2024-11-26T14:16:14,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742153_1329 (size=12301) 2024-11-26T14:16:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:15,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=881 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/bbfe5311d8c84149af544ad7585c25b4 2024-11-26T14:16:15,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/21913776787440359e5096bf24983dda is 50, key is test_row_0/C:col10/1732630574265/Put/seqid=0 2024-11-26T14:16:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742154_1330 (size=12301) 2024-11-26T14:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:15,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=881 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/21913776787440359e5096bf24983dda 2024-11-26T14:16:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/5bb094bac057446b956a9f73a7d4cd0f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/5bb094bac057446b956a9f73a7d4cd0f 2024-11-26T14:16:15,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/5bb094bac057446b956a9f73a7d4cd0f, entries=150, sequenceid=881, filesize=12.0 K 2024-11-26T14:16:15,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/bbfe5311d8c84149af544ad7585c25b4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/bbfe5311d8c84149af544ad7585c25b4 2024-11-26T14:16:15,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/bbfe5311d8c84149af544ad7585c25b4, entries=150, sequenceid=881, filesize=12.0 K 2024-11-26T14:16:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/21913776787440359e5096bf24983dda as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/21913776787440359e5096bf24983dda 2024-11-26T14:16:15,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/21913776787440359e5096bf24983dda, entries=150, sequenceid=881, filesize=12.0 K 2024-11-26T14:16:15,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 66833f8312ba730f0c055c76a8c04f74 in 1285ms, sequenceid=881, compaction requested=false 2024-11-26T14:16:15,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66833f8312ba730f0c055c76a8c04f74: 2024-11-26T14:16:15,554 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. after waiting 0 ms 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:15,555 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(2902): Flushing 66833f8312ba730f0c055c76a8c04f74 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=A 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=B 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66833f8312ba730f0c055c76a8c04f74, store=C 2024-11-26T14:16:15,555 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:15,559 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/6c5794be6cb849c18852406c307bfc92 is 50, key is test_row_1/A:col10/1732630574274/Put/seqid=0 2024-11-26T14:16:15,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742155_1331 (size=9857) 2024-11-26T14:16:15,569 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/6c5794be6cb849c18852406c307bfc92 2024-11-26T14:16:15,575 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/cf90b95f935842df8d4ae5da4154abf5 is 50, key is test_row_1/B:col10/1732630574274/Put/seqid=0 2024-11-26T14:16:15,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742156_1332 (size=9857) 2024-11-26T14:16:15,979 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/cf90b95f935842df8d4ae5da4154abf5 2024-11-26T14:16:16,003 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/01db4bc45b7947fe943bed869c51a860 is 50, key is test_row_1/C:col10/1732630574274/Put/seqid=0 2024-11-26T14:16:16,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742157_1333 (size=9857) 2024-11-26T14:16:16,407 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=886 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/01db4bc45b7947fe943bed869c51a860 2024-11-26T14:16:16,413 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/A/6c5794be6cb849c18852406c307bfc92 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/6c5794be6cb849c18852406c307bfc92 2024-11-26T14:16:16,418 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/6c5794be6cb849c18852406c307bfc92, entries=100, sequenceid=886, filesize=9.6 K 2024-11-26T14:16:16,419 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/B/cf90b95f935842df8d4ae5da4154abf5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/cf90b95f935842df8d4ae5da4154abf5 2024-11-26T14:16:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:16,425 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/cf90b95f935842df8d4ae5da4154abf5, entries=100, sequenceid=886, filesize=9.6 K 2024-11-26T14:16:16,426 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/.tmp/C/01db4bc45b7947fe943bed869c51a860 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/01db4bc45b7947fe943bed869c51a860 2024-11-26T14:16:16,432 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/01db4bc45b7947fe943bed869c51a860, entries=100, sequenceid=886, filesize=9.6 K 2024-11-26T14:16:16,433 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 66833f8312ba730f0c055c76a8c04f74 in 878ms, sequenceid=886, compaction requested=true 2024-11-26T14:16:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/93f9723af75446bab2a4b3928cd9ec1d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e78c531be35f49fd93f39a0de7694163, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/96fafa78d313423a99f4fabc48871c4c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e118470bc6ae478bb505f4dd1ad689bc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/9f47a86040ac4791887c298f2dc09b19, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/69d595fa967f41928089ea940a675ebe, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/0ade4b5eddfa4b92bdcbc6b984803c2d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/a5a2014514b54208b5137a705bc8684f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b0ef5daf33014df49255bf10a423a8fd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db] to archive 2024-11-26T14:16:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e634d20d10a644e3a6acd8c94f3bdb06 2024-11-26T14:16:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/4bf3e811efaa4737a06c767388787418 2024-11-26T14:16:16,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/7099b6202cf4491e93b42fc9d0e44077 2024-11-26T14:16:16,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/8327f3d45c034401ae2938fe71fb04a8 2024-11-26T14:16:16,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/93f9723af75446bab2a4b3928cd9ec1d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/93f9723af75446bab2a4b3928cd9ec1d 2024-11-26T14:16:16,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/29c69f97687a4b7c9269a1c4d6bb2df2 2024-11-26T14:16:16,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e78c531be35f49fd93f39a0de7694163 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e78c531be35f49fd93f39a0de7694163 2024-11-26T14:16:16,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b53a6758136540acbb9f0a01c5a3b475 2024-11-26T14:16:16,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b372ac5bef464619a65635b423259b0c 2024-11-26T14:16:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/96fafa78d313423a99f4fabc48871c4c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/96fafa78d313423a99f4fabc48871c4c 2024-11-26T14:16:16,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fcfad41990cf490c88613516d304be7e 2024-11-26T14:16:16,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e083050c497141c9ad3dd882fcb97a41 2024-11-26T14:16:16,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d874cad440214903bc50dfbaf78391d9 2024-11-26T14:16:16,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e118470bc6ae478bb505f4dd1ad689bc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/e118470bc6ae478bb505f4dd1ad689bc 2024-11-26T14:16:16,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/f907fa74c55f46faa0cb6c05238d2994 2024-11-26T14:16:16,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/2569bd7acc3e4b378a103b863e8c2a98 2024-11-26T14:16:16,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/9f47a86040ac4791887c298f2dc09b19 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/9f47a86040ac4791887c298f2dc09b19 2024-11-26T14:16:16,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/934a920c3fb0412c9aa5312f06452e8e 2024-11-26T14:16:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fc10fa60ef79470b86d40d0a21209677 2024-11-26T14:16:16,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/62b28ca1e9f64987aa3094f840154852 2024-11-26T14:16:16,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/69d595fa967f41928089ea940a675ebe to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/69d595fa967f41928089ea940a675ebe 2024-11-26T14:16:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/73c5f5c419f547dd9cb08aad590f4a83 2024-11-26T14:16:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/0ade4b5eddfa4b92bdcbc6b984803c2d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/0ade4b5eddfa4b92bdcbc6b984803c2d 2024-11-26T14:16:16,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/df11122eb7534fc089abaa6c74885bca 2024-11-26T14:16:16,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/d5762d6828d144628b12c73b076853d3 2024-11-26T14:16:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/a5a2014514b54208b5137a705bc8684f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/a5a2014514b54208b5137a705bc8684f 2024-11-26T14:16:16,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/72bae0da597040058311c6bc4e696f95 2024-11-26T14:16:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/57fbcdbedcac4137b946bc3d425fd885 2024-11-26T14:16:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/fa054e992074466984c67cce6198cd99 2024-11-26T14:16:16,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b0ef5daf33014df49255bf10a423a8fd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b0ef5daf33014df49255bf10a423a8fd 2024-11-26T14:16:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/407a7d778bc640f69ec4b098f99e9e83 2024-11-26T14:16:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/b7d2cc9c500b474c8354f0064d63a0db 2024-11-26T14:16:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/9434fc992d8f4907a6f657a16e2daacd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/4052fef8cbeb4c42970d55f57e994475, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e01e60c29ce344289fae4785fdbfb923, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f7005ea15d4d4cc19bbd032d9cff0fe9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/402c1d3f27b249e7978cbf2462c96882, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f02b6945a5bf494cb6c33ce81c989a07, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d7ed522e845c40ac82c81ce8506b708a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8c5c24e0ef444a7783554729175fcefe, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/1ff0eb4099504528b3c1d29f5b545aa3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c] to archive 2024-11-26T14:16:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:16,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e838c80501ee40108d02edd54dd17f64 2024-11-26T14:16:16,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d447c483a31d4b22a80734a051eeed5a 2024-11-26T14:16:16,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/67fd56a9f4504ff5857eea2ede57e7e0 2024-11-26T14:16:16,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/9434fc992d8f4907a6f657a16e2daacd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/9434fc992d8f4907a6f657a16e2daacd 2024-11-26T14:16:16,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/5e40c92e8d204a15bfaf6f6c5d6e33d5 2024-11-26T14:16:16,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/7d6fdc7f52d04e27b3538e5fc14a9e57 2024-11-26T14:16:16,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/4052fef8cbeb4c42970d55f57e994475 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/4052fef8cbeb4c42970d55f57e994475 2024-11-26T14:16:16,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8257bec1b9a348dcbb520bd6d85a7988 2024-11-26T14:16:16,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/567464576f974aabb2d3f643d82e37d8 2024-11-26T14:16:16,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e01e60c29ce344289fae4785fdbfb923 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/e01e60c29ce344289fae4785fdbfb923 2024-11-26T14:16:16,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3403f39a06874043866fe55745f175a1 2024-11-26T14:16:16,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/2884d3c4aedf45e58fc398467d506367 2024-11-26T14:16:16,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f7005ea15d4d4cc19bbd032d9cff0fe9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f7005ea15d4d4cc19bbd032d9cff0fe9 2024-11-26T14:16:16,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/15847a3aab644a90992b9cfc37d470ae 2024-11-26T14:16:16,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/163a30543de545fe89b29dbaee440893 2024-11-26T14:16:16,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/402c1d3f27b249e7978cbf2462c96882 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/402c1d3f27b249e7978cbf2462c96882 2024-11-26T14:16:16,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/90ed99b67da046da9be83a082ec6dd26 2024-11-26T14:16:16,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/a4c24bef7a474907b4f2c50afc7d8c2c 2024-11-26T14:16:16,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/03ef2b01f4404dcdb28bdc738bbbeac9 2024-11-26T14:16:16,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f02b6945a5bf494cb6c33ce81c989a07 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f02b6945a5bf494cb6c33ce81c989a07 2024-11-26T14:16:16,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/74b19bb6cf94461da4f82d584c82358d 2024-11-26T14:16:16,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/54538094eab04a3784323f59f43ee47a 2024-11-26T14:16:16,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d7ed522e845c40ac82c81ce8506b708a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/d7ed522e845c40ac82c81ce8506b708a 2024-11-26T14:16:16,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/f03fb5a6fb66431cb8071a5daa07859d 2024-11-26T14:16:16,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/0e510804880647b39ceaa2b22bdef54d 2024-11-26T14:16:16,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8c5c24e0ef444a7783554729175fcefe to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/8c5c24e0ef444a7783554729175fcefe 2024-11-26T14:16:16,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/728d4ba6061b44e6b18109e8f53b5671 2024-11-26T14:16:16,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/11f4f1edd05e4593b2d9a29f451936e7 2024-11-26T14:16:16,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/1ff0eb4099504528b3c1d29f5b545aa3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/1ff0eb4099504528b3c1d29f5b545aa3 2024-11-26T14:16:16,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3f30e76b5fc141a88a6ba3323d4b98ba 2024-11-26T14:16:16,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/3e46d3e8c5f642df8efcc0e217058aee 2024-11-26T14:16:16,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/aa2b6ae9d7504286a0bce04d9eb3024c 2024-11-26T14:16:16,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95eb48dc9ecb4329861c2aad8dfd219d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/eed9ef29a66145d6a6ceaac1b41e0acf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/af16eb9fe61842ad9ae8faf329d3da21, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce01047b65604de4b19e0e2f91c3ee60, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/a7dc9502041f416796a66ea21398f96a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95a5b929f3574837a62d38223ea42b68, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/34421732fdb049f58ed09e1d0f71bf67, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/59c8bd3262cb4baebc4cc60b5c309ad1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/369bb7e2a2314cc4957a60d7107daac6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d] to archive 2024-11-26T14:16:16,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:16,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/d8548b9a7cdc485d91842c24c80dfddb 2024-11-26T14:16:16,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fb85ae58aea3484db2b6442580c629fa 2024-11-26T14:16:16,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/381c0c8f495b4b79aa50451876b2e554 2024-11-26T14:16:16,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95eb48dc9ecb4329861c2aad8dfd219d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95eb48dc9ecb4329861c2aad8dfd219d 2024-11-26T14:16:16,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f0a6228dd0d2413abd871066d4b40e02 2024-11-26T14:16:16,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce6288c492df4c9495fbfd56ca044455 2024-11-26T14:16:16,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/eed9ef29a66145d6a6ceaac1b41e0acf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/eed9ef29a66145d6a6ceaac1b41e0acf 2024-11-26T14:16:16,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/0e97bff1976a422791a4ed4e0c9fac7b 2024-11-26T14:16:16,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/848b8043b3e64847a5ae3542675ab888 2024-11-26T14:16:16,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/af16eb9fe61842ad9ae8faf329d3da21 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/af16eb9fe61842ad9ae8faf329d3da21 2024-11-26T14:16:16,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/71b077e1e63348ecac5eebd04f3a8a56 2024-11-26T14:16:16,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b5d8c8eea11c49d794b3351c9b7f1eaf 2024-11-26T14:16:16,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce01047b65604de4b19e0e2f91c3ee60 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/ce01047b65604de4b19e0e2f91c3ee60 2024-11-26T14:16:16,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/40261651f4e44852bdf38937488514c8 2024-11-26T14:16:16,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b905ce7fd1784b9e8362ecddd6d3fa7f 2024-11-26T14:16:16,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/a7dc9502041f416796a66ea21398f96a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/a7dc9502041f416796a66ea21398f96a 2024-11-26T14:16:16,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/2413054938cb4d50bfcd518a4d27faf6 2024-11-26T14:16:16,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f2b366b5a2c44963b9812137d9405ebc 2024-11-26T14:16:16,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/72e94b7dd74140199d00460e5a4e5135 2024-11-26T14:16:16,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95a5b929f3574837a62d38223ea42b68 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/95a5b929f3574837a62d38223ea42b68 2024-11-26T14:16:16,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b873cefa8b324b2a877996250ec7068f 2024-11-26T14:16:16,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fc09cd4246cd4dd0904eb9aee1738769 2024-11-26T14:16:16,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/34421732fdb049f58ed09e1d0f71bf67 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/34421732fdb049f58ed09e1d0f71bf67 2024-11-26T14:16:16,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/9ac66af9dfc5426caeb0f8ddc138813f 2024-11-26T14:16:16,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/e517d68f0975409a96e9efabf0837089 2024-11-26T14:16:16,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/59c8bd3262cb4baebc4cc60b5c309ad1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/59c8bd3262cb4baebc4cc60b5c309ad1 2024-11-26T14:16:16,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/8b66617a1f64477c96ac92615b7bd24d 2024-11-26T14:16:16,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/f81f526844524fcc86db465539d5f439 2024-11-26T14:16:16,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/369bb7e2a2314cc4957a60d7107daac6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/369bb7e2a2314cc4957a60d7107daac6 2024-11-26T14:16:16,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/4a847483346d4bb79bec94e064a48980 2024-11-26T14:16:16,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/510e20d952ae44058d92c18739fbc76b 2024-11-26T14:16:16,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/fe4acd3f192c436f951bb92d0c3d982d 2024-11-26T14:16:16,624 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/recovered.edits/889.seqid, newMaxSeqId=889, maxSeqId=1 2024-11-26T14:16:16,624 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. 2024-11-26T14:16:16,624 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1676): Region close journal for 66833f8312ba730f0c055c76a8c04f74: Waiting for close lock at 1732630574440Running coprocessor pre-close hooks at 1732630574440Disabling compacts and flushes for region at 1732630574440Disabling writes for close at 1732630575555 (+1115 ms)Obtaining lock to block concurrent updates at 1732630575555Preparing flush snapshotting stores in 66833f8312ba730f0c055c76a8c04f74 at 1732630575555Finished memstore snapshotting TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732630575555Flushing stores of TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74. at 1732630575556 (+1 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/A: creating writer at 1732630575556Flushing 66833f8312ba730f0c055c76a8c04f74/A: appending metadata at 1732630575559 (+3 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/A: closing flushed file at 1732630575559Flushing 66833f8312ba730f0c055c76a8c04f74/B: creating writer at 1732630575573 (+14 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/B: appending metadata at 1732630575575 (+2 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/B: closing flushed file at 1732630575575Flushing 66833f8312ba730f0c055c76a8c04f74/C: creating writer at 1732630575984 (+409 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/C: appending metadata at 1732630576002 (+18 ms)Flushing 66833f8312ba730f0c055c76a8c04f74/C: closing flushed file at 1732630576002Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bfa2cfd: reopening flushed file at 1732630576412 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a450e6a: reopening flushed file at 1732630576418 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59db0de2: reopening flushed file at 1732630576425 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 66833f8312ba730f0c055c76a8c04f74 in 878ms, sequenceid=886, compaction requested=true at 1732630576433 (+8 ms)Writing region close event to WAL at 1732630576620 (+187 ms)Running coprocessor post-close hooks at 1732630576624 (+4 ms)Closed at 1732630576624 2024-11-26T14:16:16,626 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(157): Closed 66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:16,626 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=66833f8312ba730f0c055c76a8c04f74, regionState=CLOSED 2024-11-26T14:16:16,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:16,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T14:16:16,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseRegionProcedure 66833f8312ba730f0c055c76a8c04f74, server=19945b63c2b4,40025,1732630490995 in 2.3410 sec 2024-11-26T14:16:16,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-26T14:16:16,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66833f8312ba730f0c055c76a8c04f74, UNASSIGN in 2.3460 sec 2024-11-26T14:16:16,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T14:16:16,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3490 sec 2024-11-26T14:16:16,635 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630576635"}]},"ts":"1732630576635"} 2024-11-26T14:16:16,637 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:16:16,637 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:16:16,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3600 sec 2024-11-26T14:16:18,385 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:16:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T14:16:18,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:18,434 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:16:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,437 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,438 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=72, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T14:16:18,444 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:18,449 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/recovered.edits] 2024-11-26T14:16:18,451 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/5bb094bac057446b956a9f73a7d4cd0f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/5bb094bac057446b956a9f73a7d4cd0f 2024-11-26T14:16:18,452 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/6c5794be6cb849c18852406c307bfc92 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/6c5794be6cb849c18852406c307bfc92 2024-11-26T14:16:18,453 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/c1959fa1add3431babe3742973032409 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/A/c1959fa1add3431babe3742973032409 2024-11-26T14:16:18,454 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/850d6450281d4cabaa0445939e87ec96 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/850d6450281d4cabaa0445939e87ec96 2024-11-26T14:16:18,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/bbfe5311d8c84149af544ad7585c25b4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/bbfe5311d8c84149af544ad7585c25b4 2024-11-26T14:16:18,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/cf90b95f935842df8d4ae5da4154abf5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/B/cf90b95f935842df8d4ae5da4154abf5 2024-11-26T14:16:18,459 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/01db4bc45b7947fe943bed869c51a860 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/01db4bc45b7947fe943bed869c51a860 2024-11-26T14:16:18,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/21913776787440359e5096bf24983dda to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/21913776787440359e5096bf24983dda 2024-11-26T14:16:18,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b7e69c03ecaf4069ae3107e114a63885 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/C/b7e69c03ecaf4069ae3107e114a63885 2024-11-26T14:16:18,465 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/recovered.edits/889.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74/recovered.edits/889.seqid 2024-11-26T14:16:18,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/66833f8312ba730f0c055c76a8c04f74 2024-11-26T14:16:18,467 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:16:18,469 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=72, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,473 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:16:18,475 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:16:18,477 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=72, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,477 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:16:18,477 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630578477"}]},"ts":"9223372036854775807"} 2024-11-26T14:16:18,480 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:16:18,480 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66833f8312ba730f0c055c76a8c04f74, NAME => 'TestAcidGuarantees,,1732630551396.66833f8312ba730f0c055c76a8c04f74.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:16:18,480 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:16:18,480 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630578480"}]},"ts":"9223372036854775807"} 2024-11-26T14:16:18,482 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:16:18,483 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=72, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-26T14:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T14:16:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:16:18,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:18,557 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251 (was 252), OpenFileDescriptor=457 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1055 (was 1092), ProcessCount=11 (was 11), AvailableMemoryMB=6105 (was 3672) - AvailableMemoryMB LEAK? - 2024-11-26T14:16:18,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1055, ProcessCount=11, AvailableMemoryMB=6104 2024-11-26T14:16:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:16:18,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:16:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:18,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:16:18,573 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:18,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 73 2024-11-26T14:16:18,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:16:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:18,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742158_1334 (size=1061) 2024-11-26T14:16:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:19,005 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:16:19,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742159_1335 (size=53) 2024-11-26T14:16:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:19,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:16:19,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 003c34ee972d42fbe3c2e22bf0ce4d43, disabling compactions & flushes 2024-11-26T14:16:19,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. after waiting 0 ms 2024-11-26T14:16:19,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,428 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 003c34ee972d42fbe3c2e22bf0ce4d43: Waiting for close lock at 1732630579427Disabling compacts and flushes for region at 1732630579427Disabling writes for close at 1732630579427Writing region close event to WAL at 1732630579428 (+1 ms)Closed at 1732630579428 2024-11-26T14:16:19,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:16:19,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630579429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630579429"}]},"ts":"1732630579429"} 2024-11-26T14:16:19,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:16:19,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:16:19,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630579432"}]},"ts":"1732630579432"} 2024-11-26T14:16:19,434 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:16:19,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, ASSIGN}] 2024-11-26T14:16:19,437 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, ASSIGN 2024-11-26T14:16:19,438 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:16:19,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:19,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, ASSIGN because future has completed 2024-11-26T14:16:19,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:19,745 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,745 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:16:19,745 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,745 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:16:19,745 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,745 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,746 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,747 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:19,747 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName A 2024-11-26T14:16:19,747 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:19,748 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:19,748 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,748 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:19,749 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName B 2024-11-26T14:16:19,749 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:19,749 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:19,749 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,750 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:19,750 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName C 2024-11-26T14:16:19,750 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:19,750 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:19,750 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,750 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,751 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,751 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,752 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,752 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,752 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:16:19,753 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,755 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:16:19,755 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 003c34ee972d42fbe3c2e22bf0ce4d43; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64241971, jitterRate=-0.04272003471851349}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:16:19,755 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:19,756 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 003c34ee972d42fbe3c2e22bf0ce4d43: Running coprocessor pre-open hook at 1732630579745Writing region info on filesystem at 1732630579745Initializing all the Stores at 1732630579746 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630579746Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630579746Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630579746Cleaning up temporary data from old regions at 1732630579752 (+6 ms)Running coprocessor post-open hooks at 1732630579755 (+3 ms)Region opened successfully at 1732630579756 (+1 ms) 2024-11-26T14:16:19,756 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., pid=75, masterSystemTime=1732630579742 2024-11-26T14:16:19,758 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,758 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:19,758 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:19,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:19,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T14:16:19,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 in 170 msec 2024-11-26T14:16:19,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-26T14:16:19,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, ASSIGN in 327 msec 2024-11-26T14:16:19,763 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:16:19,763 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630579763"}]},"ts":"1732630579763"} 2024-11-26T14:16:19,764 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:16:19,765 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:16:19,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-26T14:16:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-26T14:16:20,723 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:20,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:20,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:20,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:20,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302fea40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:20,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4277c86a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:20,730 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:20,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:20,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:16:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:16:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@31e9334a 2024-11-26T14:16:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:16:20,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:16:20,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:16:20,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:16:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:20,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742160_1336 (size=1097) 2024-11-26T14:16:21,159 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T14:16:21,159 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T14:16:21,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:16:21,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE}] 2024-11-26T14:16:21,167 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE 2024-11-26T14:16:21,168 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE because future has completed 2024-11-26T14:16:21,170 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:16:21,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:21,322 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,322 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:16:21,322 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 003c34ee972d42fbe3c2e22bf0ce4d43, disabling compactions & flushes 2024-11-26T14:16:21,323 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,323 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,323 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. after waiting 0 ms 2024-11-26T14:16:21,323 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,327 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T14:16:21,327 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,327 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 003c34ee972d42fbe3c2e22bf0ce4d43: Waiting for close lock at 1732630581322Running coprocessor pre-close hooks at 1732630581322Disabling compacts and flushes for region at 1732630581322Disabling writes for close at 1732630581323 (+1 ms)Writing region close event to WAL at 1732630581323Running coprocessor post-close hooks at 1732630581327 (+4 ms)Closed at 1732630581327 2024-11-26T14:16:21,327 WARN [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionServer(3299): Not adding moved region record: 003c34ee972d42fbe3c2e22bf0ce4d43 to self. 2024-11-26T14:16:21,329 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=CLOSED 2024-11-26T14:16:21,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:21,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T14:16:21,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 in 161 msec 2024-11-26T14:16:21,333 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE; state=CLOSED, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=true 2024-11-26T14:16:21,484 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE because future has completed 2024-11-26T14:16:21,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:21,641 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,641 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:16:21,641 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,641 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:16:21,641 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,642 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,643 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,643 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:21,644 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName A 2024-11-26T14:16:21,645 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:21,645 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:21,645 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,646 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:21,646 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName B 2024-11-26T14:16:21,646 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:21,646 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:21,647 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,647 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:21,647 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 003c34ee972d42fbe3c2e22bf0ce4d43 columnFamilyName C 2024-11-26T14:16:21,647 DEBUG [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:21,647 INFO [StoreOpener-003c34ee972d42fbe3c2e22bf0ce4d43-1 {}] regionserver.HStore(327): Store=003c34ee972d42fbe3c2e22bf0ce4d43/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:21,648 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,648 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,648 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,649 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,651 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,651 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,651 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:16:21,653 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,653 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 003c34ee972d42fbe3c2e22bf0ce4d43; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70315687, jitterRate=0.04778538644313812}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:16:21,654 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,654 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 003c34ee972d42fbe3c2e22bf0ce4d43: Running coprocessor pre-open hook at 1732630581642Writing region info on filesystem at 1732630581642Initializing all the Stores at 1732630581642Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630581642Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630581642Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630581642Cleaning up temporary data from old regions at 1732630581651 (+9 ms)Running coprocessor post-open hooks at 1732630581654 (+3 ms)Region opened successfully at 1732630581654 2024-11-26T14:16:21,655 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., pid=80, masterSystemTime=1732630581638 2024-11-26T14:16:21,657 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,657 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,657 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=OPEN, openSeqNum=5, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:21,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=78 2024-11-26T14:16:21,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 in 173 msec 2024-11-26T14:16:21,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-26T14:16:21,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, REOPEN/MOVE in 495 msec 2024-11-26T14:16:21,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T14:16:21,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-26T14:16:21,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-26T14:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T14:16:21,668 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:21,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,669 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ce973f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d53f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,677 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18391623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b029208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,681 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7d167c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,682 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4989ae95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c193e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1620e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140efc75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,690 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0fe2c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798d329f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d4fc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd04393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36339697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b522f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c2d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,707 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9d5a54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a24c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:16:21,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:16:21,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598b4af8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:16:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:16:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:21,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:16:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34531e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:16:21,722 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-11-26T14:16:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:21,725 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:21,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:21,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,726 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,727 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,728 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,728 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,730 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,731 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,731 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,732 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,736 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:16:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:16:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:16:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:16:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:16:21,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630641832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630581810, max timestamp=1732630581849Num uniques 859; 2024-11-26T14:16:21,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630581810, max timestamp=1732630581849Num uniques 859; 2024-11-26T14:16:21,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:16:21,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630581810, max timestamp=1732630581849Num uniques 859; 2024-11-26T14:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:16:21,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126457fc7b00cb44764bdc4c96c575920a7_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630581755/Put/seqid=0 2024-11-26T14:16:21,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:21,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:21,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742161_1337 (size=14594) 2024-11-26T14:16:21,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:21,933 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126457fc7b00cb44764bdc4c96c575920a7_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126457fc7b00cb44764bdc4c96c575920a7_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:21,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/30866d159e8d4e96b96679324670e004, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:21,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/30866d159e8d4e96b96679324670e004 is 175, key is test_row_0/A:col10/1732630581755/Put/seqid=0 2024-11-26T14:16:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630641928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630641928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630641949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630641951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630641953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:21,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:21,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742162_1338 (size=39549) 2024-11-26T14:16:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:22,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630642064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630642077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630642084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630642084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,154 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:16:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630642198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630642283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630642297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630642302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630642304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:22,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/30866d159e8d4e96b96679324670e004 2024-11-26T14:16:22,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/cc30610b82dd42919a84051574002684 is 50, key is test_row_0/B:col10/1732630581755/Put/seqid=0 2024-11-26T14:16:22,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742163_1339 (size=12001) 2024-11-26T14:16:22,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630642523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630642604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630642617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630642623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630642628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:22,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:22,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:22,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/cc30610b82dd42919a84051574002684 2024-11-26T14:16:22,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/aa82f34b6fca46fbb28376ee73892eb3 is 50, key is test_row_0/C:col10/1732630581755/Put/seqid=0 2024-11-26T14:16:22,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742164_1340 (size=12001) 2024-11-26T14:16:22,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/aa82f34b6fca46fbb28376ee73892eb3 2024-11-26T14:16:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/30866d159e8d4e96b96679324670e004 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004 2024-11-26T14:16:22,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004, entries=200, sequenceid=15, filesize=38.6 K 2024-11-26T14:16:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/cc30610b82dd42919a84051574002684 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684 2024-11-26T14:16:22,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T14:16:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/aa82f34b6fca46fbb28376ee73892eb3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3 2024-11-26T14:16:23,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T14:16:23,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1238ms, sequenceid=15, compaction requested=false 2024-11-26T14:16:23,002 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T14:16:23,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:23,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-26T14:16:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:23,008 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8f22a2a01674893b0c3b85089a1811a_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630581921/Put/seqid=0 2024-11-26T14:16:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742165_1341 (size=12154) 2024-11-26T14:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:23,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:23,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630643120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630643124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630643144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630643145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630643154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630643245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:23,458 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8f22a2a01674893b0c3b85089a1811a_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8f22a2a01674893b0c3b85089a1811a_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:23,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7618b758b01242188edb5e64db94f78d, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7618b758b01242188edb5e64db94f78d is 175, key is test_row_0/A:col10/1732630581921/Put/seqid=0 2024-11-26T14:16:23,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630643463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742166_1342 (size=30955) 2024-11-26T14:16:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630643774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:23,894 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7618b758b01242188edb5e64db94f78d 2024-11-26T14:16:23,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/7aa766511f3145639026c8cc42372d09 is 50, key is test_row_0/B:col10/1732630581921/Put/seqid=0 2024-11-26T14:16:23,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742167_1343 (size=12001) 2024-11-26T14:16:23,951 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/7aa766511f3145639026c8cc42372d09 2024-11-26T14:16:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/f2c2cd753d694d2381a0966ad933a30f is 50, key is test_row_0/C:col10/1732630581921/Put/seqid=0 2024-11-26T14:16:23,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742168_1344 (size=12001) 2024-11-26T14:16:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630644153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:24,164 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630644163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:24,174 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630644176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:24,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630644176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:24,191 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:24,191 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:24,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630644293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:24,393 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/f2c2cd753d694d2381a0966ad933a30f 2024-11-26T14:16:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7618b758b01242188edb5e64db94f78d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d 2024-11-26T14:16:24,410 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d, entries=150, sequenceid=49, filesize=30.2 K 2024-11-26T14:16:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/7aa766511f3145639026c8cc42372d09 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09 2024-11-26T14:16:24,416 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:16:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/f2c2cd753d694d2381a0966ad933a30f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f 2024-11-26T14:16:24,434 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:16:24,435 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1428ms, sequenceid=49, compaction requested=false 2024-11-26T14:16:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-26T14:16:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-11-26T14:16:24,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-26T14:16:24,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7220 sec 2024-11-26T14:16:24,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 2.7350 sec 2024-11-26T14:16:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:25,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:25,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f3be4b30308b490c8b64e544e5693141_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:25,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742169_1345 (size=14594) 2024-11-26T14:16:25,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630645563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630645683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:25,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f3be4b30308b490c8b64e544e5693141_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f3be4b30308b490c8b64e544e5693141_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:25,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/d4b08dbb4ae84f3498415d3a850fc4ac, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:25,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/d4b08dbb4ae84f3498415d3a850fc4ac is 175, key is test_row_0/A:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:25,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742170_1346 (size=39549) 2024-11-26T14:16:25,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/d4b08dbb4ae84f3498415d3a850fc4ac 2024-11-26T14:16:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-26T14:16:25,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:25,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/d5985fcde073467890305a3df3e93eb1 is 50, key is test_row_0/B:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:25,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-26T14:16:25,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:25,902 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:25,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630645903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:25,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742171_1347 (size=12001) 2024-11-26T14:16:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:26,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630646184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,191 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630646193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,203 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630646204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,208 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630646214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:26,223 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630646233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:26,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/d5985fcde073467890305a3df3e93eb1 2024-11-26T14:16:26,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/6148e8d8ca3c499aa0e1b02d94c25d4b is 50, key is test_row_0/C:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:26,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742172_1348 (size=12001) 2024-11-26T14:16:26,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630646753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:26,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:26,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:26,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:27,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/6148e8d8ca3c499aa0e1b02d94c25d4b 2024-11-26T14:16:27,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/d4b08dbb4ae84f3498415d3a850fc4ac as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac 2024-11-26T14:16:27,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac, entries=200, sequenceid=61, filesize=38.6 K 2024-11-26T14:16:27,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/d5985fcde073467890305a3df3e93eb1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1 2024-11-26T14:16:27,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T14:16:27,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/6148e8d8ca3c499aa0e1b02d94c25d4b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b 2024-11-26T14:16:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:27,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T14:16:27,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1718ms, sequenceid=61, compaction requested=true 2024-11-26T14:16:27,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:27,048 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:27,048 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:27,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:27,049 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:27,049 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:27,049 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/B is initiating minor compaction (all files) 2024-11-26T14:16:27,049 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/A is initiating minor compaction (all files) 2024-11-26T14:16:27,049 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/B in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:27,049 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/A in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:27,049 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=107.5 K 2024-11-26T14:16:27,049 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.2 K 2024-11-26T14:16:27,050 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac] 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting cc30610b82dd42919a84051574002684, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630581742 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30866d159e8d4e96b96679324670e004, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630581742 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa766511f3145639026c8cc42372d09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630581921 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7618b758b01242188edb5e64db94f78d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630581921 2024-11-26T14:16:27,050 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d5985fcde073467890305a3df3e93eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:27,051 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b08dbb4ae84f3498415d3a850fc4ac, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:27,060 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:27,064 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:27,065 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/3df3a9bb58f543038b04100393a9f60f is 50, key is test_row_0/B:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:27,089 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126bf1c53622870403690121fa23ddb79b6_003c34ee972d42fbe3c2e22bf0ce4d43 store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:27,093 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126bf1c53622870403690121fa23ddb79b6_003c34ee972d42fbe3c2e22bf0ce4d43, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:27,093 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bf1c53622870403690121fa23ddb79b6_003c34ee972d42fbe3c2e22bf0ce4d43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:27,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-26T14:16:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:27,133 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:27,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742173_1349 (size=12104) 2024-11-26T14:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742174_1350 (size=4469) 2024-11-26T14:16:27,143 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:16:27,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a3158c10b2de4c96a0a9635f4885dcb7_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630585539/Put/seqid=0 2024-11-26T14:16:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742175_1351 (size=12154) 2024-11-26T14:16:27,535 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#A#compaction#296 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:27,536 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b185a1da02bd4773948a5a48ce55ed24 is 175, key is test_row_0/A:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:27,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742176_1352 (size=31058) 2024-11-26T14:16:27,546 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/3df3a9bb58f543038b04100393a9f60f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/3df3a9bb58f543038b04100393a9f60f 2024-11-26T14:16:27,555 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b185a1da02bd4773948a5a48ce55ed24 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24 2024-11-26T14:16:27,555 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/B of 003c34ee972d42fbe3c2e22bf0ce4d43 into 3df3a9bb58f543038b04100393a9f60f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:27,556 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:27,556 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/B, priority=13, startTime=1732630587048; duration=0sec 2024-11-26T14:16:27,556 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:27,556 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:B 2024-11-26T14:16:27,556 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:27,558 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:27,559 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/C is initiating minor compaction (all files) 2024-11-26T14:16:27,559 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/C in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:27,559 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.2 K 2024-11-26T14:16:27,559 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa82f34b6fca46fbb28376ee73892eb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630581742 2024-11-26T14:16:27,560 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c2cd753d694d2381a0966ad933a30f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630581921 2024-11-26T14:16:27,560 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6148e8d8ca3c499aa0e1b02d94c25d4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:27,563 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/A of 003c34ee972d42fbe3c2e22bf0ce4d43 into b185a1da02bd4773948a5a48ce55ed24(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:27,563 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:27,563 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/A, priority=13, startTime=1732630587048; duration=0sec 2024-11-26T14:16:27,563 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:27,563 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:A 2024-11-26T14:16:27,572 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:27,572 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/dcb7b1fa2c1e4660a002f7154b714384 is 50, key is test_row_0/C:col10/1732630583096/Put/seqid=0 2024-11-26T14:16:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742177_1353 (size=12104) 2024-11-26T14:16:27,603 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/dcb7b1fa2c1e4660a002f7154b714384 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/dcb7b1fa2c1e4660a002f7154b714384 2024-11-26T14:16:27,606 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/C of 003c34ee972d42fbe3c2e22bf0ce4d43 into dcb7b1fa2c1e4660a002f7154b714384(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:27,607 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:27,607 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/C, priority=13, startTime=1732630587048; duration=0sec 2024-11-26T14:16:27,607 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:27,607 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:C 2024-11-26T14:16:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:27,617 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a3158c10b2de4c96a0a9635f4885dcb7_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3158c10b2de4c96a0a9635f4885dcb7_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:27,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/930de1336b004020aed98830f207e548, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:27,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/930de1336b004020aed98830f207e548 is 175, key is test_row_0/A:col10/1732630585539/Put/seqid=0 2024-11-26T14:16:27,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742178_1354 (size=30955) 2024-11-26T14:16:27,627 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/930de1336b004020aed98830f207e548 2024-11-26T14:16:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/ab039399f80240a28db0133254fb506e is 50, key is test_row_0/B:col10/1732630585539/Put/seqid=0 2024-11-26T14:16:27,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742179_1355 (size=12001) 2024-11-26T14:16:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:27,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:27,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630647827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:27,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630647942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:28,037 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/ab039399f80240a28db0133254fb506e 2024-11-26T14:16:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/5b90da77e83844e0b9cd43d8306684b7 is 50, key is test_row_0/C:col10/1732630585539/Put/seqid=0 2024-11-26T14:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:28,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742180_1356 (size=12001) 2024-11-26T14:16:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630648163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:28,462 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/5b90da77e83844e0b9cd43d8306684b7 2024-11-26T14:16:28,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/930de1336b004020aed98830f207e548 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548 2024-11-26T14:16:28,469 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548, entries=150, sequenceid=81, filesize=30.2 K 2024-11-26T14:16:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/ab039399f80240a28db0133254fb506e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e 2024-11-26T14:16:28,474 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e, entries=150, sequenceid=81, filesize=11.7 K 2024-11-26T14:16:28,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/5b90da77e83844e0b9cd43d8306684b7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7 2024-11-26T14:16:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:28,485 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7, entries=150, sequenceid=81, filesize=11.7 K 2024-11-26T14:16:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630648483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:28,486 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1353ms, sequenceid=81, compaction requested=false 2024-11-26T14:16:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:28,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-26T14:16:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-11-26T14:16:28,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-26T14:16:28,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5860 sec 2024-11-26T14:16:28,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 2.5940 sec 2024-11-26T14:16:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:28,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:29,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126094070aa7d694edfbc6b5f3e6804c6d4_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:29,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742181_1357 (size=14594) 2024-11-26T14:16:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630649182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630649304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:29,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:29,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126094070aa7d694edfbc6b5f3e6804c6d4_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094070aa7d694edfbc6b5f3e6804c6d4_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:29,473 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/0d4b61b2483041f9a7ed35f3c3ba0cbf, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:29,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/0d4b61b2483041f9a7ed35f3c3ba0cbf is 175, key is test_row_0/A:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:29,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742182_1358 (size=39549) 2024-11-26T14:16:29,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630649523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:29,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630649833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:29,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:29,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:29,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/0d4b61b2483041f9a7ed35f3c3ba0cbf 2024-11-26T14:16:29,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/5a3be68819f94e6eb9dbd1ce55cfa72a is 50, key is test_row_0/B:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:29,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742183_1359 (size=12001) 2024-11-26T14:16:29,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/5a3be68819f94e6eb9dbd1ce55cfa72a 2024-11-26T14:16:30,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d1116e9dfce54a469e4a194c27f851ba is 50, key is test_row_0/C:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-26T14:16:30,065 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:30,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-26T14:16:30,069 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:30,070 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:30,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742184_1360 (size=12001) 2024-11-26T14:16:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:30,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T14:16:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630650233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630650233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:30,247 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:30,248 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630650233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:30,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:30,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630650236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:30,249 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:30,249 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 003c34ee972d42fbe3c2e22bf0ce4d43 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8321 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:16:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630650353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:30,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T14:16:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:30,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d1116e9dfce54a469e4a194c27f851ba 2024-11-26T14:16:30,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/0d4b61b2483041f9a7ed35f3c3ba0cbf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf 2024-11-26T14:16:30,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf, entries=200, sequenceid=97, filesize=38.6 K 2024-11-26T14:16:30,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/5a3be68819f94e6eb9dbd1ce55cfa72a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a 2024-11-26T14:16:30,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T14:16:30,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d1116e9dfce54a469e4a194c27f851ba as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba 2024-11-26T14:16:30,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T14:16:30,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1518ms, sequenceid=97, compaction requested=true 2024-11-26T14:16:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:30,513 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:30,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:30,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:30,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:30,516 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:30,516 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/A is initiating minor compaction (all files) 2024-11-26T14:16:30,516 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/A in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,516 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=99.2 K 2024-11-26T14:16:30,516 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,516 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf] 2024-11-26T14:16:30,517 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:30,517 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/B is initiating minor compaction (all files) 2024-11-26T14:16:30,517 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/B in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,517 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/3df3a9bb58f543038b04100393a9f60f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.3 K 2024-11-26T14:16:30,517 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b185a1da02bd4773948a5a48ce55ed24, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:30,517 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3df3a9bb58f543038b04100393a9f60f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:30,518 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ab039399f80240a28db0133254fb506e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732630585525 2024-11-26T14:16:30,518 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930de1336b004020aed98830f207e548, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732630585525 2024-11-26T14:16:30,518 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a3be68819f94e6eb9dbd1ce55cfa72a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:30,518 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d4b61b2483041f9a7ed35f3c3ba0cbf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:30,529 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:30,539 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:30,539 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/a89c2b32e6c741c68ee5747aa708f9f7 is 50, key is test_row_0/B:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:30,543 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126cbc9ed36b7c345baa8a3cdf02822f960_003c34ee972d42fbe3c2e22bf0ce4d43 store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:30,545 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126cbc9ed36b7c345baa8a3cdf02822f960_003c34ee972d42fbe3c2e22bf0ce4d43, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:30,545 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cbc9ed36b7c345baa8a3cdf02822f960_003c34ee972d42fbe3c2e22bf0ce4d43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:30,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-26T14:16:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:30,547 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:30,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742185_1361 (size=12207) 2024-11-26T14:16:30,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742186_1362 (size=4469) 2024-11-26T14:16:30,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec9e5385c14744bf9e4b6b70aa9d15cc_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630589175/Put/seqid=0 2024-11-26T14:16:30,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742187_1363 (size=12154) 2024-11-26T14:16:30,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:30,601 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ec9e5385c14744bf9e4b6b70aa9d15cc_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec9e5385c14744bf9e4b6b70aa9d15cc_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:30,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/933d1209b28d45aba9dde8118f1840ef, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:30,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/933d1209b28d45aba9dde8118f1840ef is 175, key is test_row_0/A:col10/1732630589175/Put/seqid=0 2024-11-26T14:16:30,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742188_1364 (size=30955) 2024-11-26T14:16:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:37,180 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1171789504_22 at /127.0.0.1:50494 [Receiving block BP-17547768-172.17.0.3-1732630485263:blk_1073741833_1009] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 5801ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data1/, blockId=1073741833, seqno=1826 2024-11-26T14:16:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:37,185 INFO [AsyncFSWAL-0-hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc-prefix:19945b63c2b4,40025,1732630490995 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5807 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK]] 2024-11-26T14:16:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:37,186 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/933d1209b28d45aba9dde8118f1840ef 2024-11-26T14:16:37,188 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#A#compaction#305 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:37,189 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/dbeb4de6c81f40c499df6bdc6e560c3b is 175, key is test_row_0/A:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:37,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742189_1365 (size=31161) 2024-11-26T14:16:37,214 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/a89c2b32e6c741c68ee5747aa708f9f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/a89c2b32e6c741c68ee5747aa708f9f7 2024-11-26T14:16:37,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/fb220024389a4e20b2bc790d059bc9b6 is 50, key is test_row_0/B:col10/1732630589175/Put/seqid=0 2024-11-26T14:16:37,226 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/B of 003c34ee972d42fbe3c2e22bf0ce4d43 into a89c2b32e6c741c68ee5747aa708f9f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 6sec to execute. 2024-11-26T14:16:37,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:37,226 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/B, priority=13, startTime=1732630590513; duration=6sec 2024-11-26T14:16:37,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:37,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:B 2024-11-26T14:16:37,226 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:37,232 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:37,232 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/C is initiating minor compaction (all files) 2024-11-26T14:16:37,232 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/C in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:37,232 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/dcb7b1fa2c1e4660a002f7154b714384, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.3 K 2024-11-26T14:16:37,233 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb7b1fa2c1e4660a002f7154b714384, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732630583096 2024-11-26T14:16:37,234 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b90da77e83844e0b9cd43d8306684b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732630585525 2024-11-26T14:16:37,234 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d1116e9dfce54a469e4a194c27f851ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:37,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742190_1366 (size=12001) 2024-11-26T14:16:37,256 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/fb220024389a4e20b2bc790d059bc9b6 2024-11-26T14:16:37,296 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:37,296 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/a25bdf17e02a4cf7942ecf66ec2eb58f is 50, key is test_row_0/C:col10/1732630588994/Put/seqid=0 2024-11-26T14:16:37,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/c019235c448645dba15060916d507411 is 50, key is test_row_0/C:col10/1732630589175/Put/seqid=0 2024-11-26T14:16:37,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742191_1367 (size=12207) 2024-11-26T14:16:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630657323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:37,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742192_1368 (size=12001) 2024-11-26T14:16:37,343 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/c019235c448645dba15060916d507411 2024-11-26T14:16:37,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/933d1209b28d45aba9dde8118f1840ef as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef 2024-11-26T14:16:37,352 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef, entries=150, sequenceid=116, filesize=30.2 K 2024-11-26T14:16:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/fb220024389a4e20b2bc790d059bc9b6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6 2024-11-26T14:16:37,358 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T14:16:37,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/c019235c448645dba15060916d507411 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411 2024-11-26T14:16:37,363 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T14:16:37,366 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 6820ms, sequenceid=116, compaction requested=false 2024-11-26T14:16:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-26T14:16:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-26T14:16:37,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-26T14:16:37,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 7.2990 sec 2024-11-26T14:16:37,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 7.3050 sec 2024-11-26T14:16:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:37,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:37,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:37,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112663e53a7d9d554b2796c4a03894426a89_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:37,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742193_1369 (size=14694) 2024-11-26T14:16:37,606 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/dbeb4de6c81f40c499df6bdc6e560c3b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b 2024-11-26T14:16:37,611 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/A of 003c34ee972d42fbe3c2e22bf0ce4d43 into dbeb4de6c81f40c499df6bdc6e560c3b(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 7sec to execute. 2024-11-26T14:16:37,611 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:37,611 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/A, priority=13, startTime=1732630590513; duration=7sec 2024-11-26T14:16:37,611 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:37,611 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:A 2024-11-26T14:16:37,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630657634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:37,742 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/a25bdf17e02a4cf7942ecf66ec2eb58f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/a25bdf17e02a4cf7942ecf66ec2eb58f 2024-11-26T14:16:37,745 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/C of 003c34ee972d42fbe3c2e22bf0ce4d43 into a25bdf17e02a4cf7942ecf66ec2eb58f(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:37,745 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:37,746 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/C, priority=13, startTime=1732630590513; duration=0sec 2024-11-26T14:16:37,746 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:37,746 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:C 2024-11-26T14:16:37,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630657743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:37,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:37,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112663e53a7d9d554b2796c4a03894426a89_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112663e53a7d9d554b2796c4a03894426a89_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:37,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7d3ef766c64644998691bbe31ab00f33, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:37,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7d3ef766c64644998691bbe31ab00f33 is 175, key is test_row_0/A:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:37,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742194_1370 (size=39649) 2024-11-26T14:16:37,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630657963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T14:16:38,263 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:38,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-26T14:16:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T14:16:38,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:38,266 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:38,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:38,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7d3ef766c64644998691bbe31ab00f33 2024-11-26T14:16:38,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/43e86e6225a64d418f2aff5f025c32fb is 50, key is test_row_0/B:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:38,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630658273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742195_1371 (size=12051) 2024-11-26T14:16:38,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/43e86e6225a64d418f2aff5f025c32fb 2024-11-26T14:16:38,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/fbb6026c0516492b886d518c23ac88e3 is 50, key is test_row_0/C:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:38,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742196_1372 (size=12051) 2024-11-26T14:16:38,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/fbb6026c0516492b886d518c23ac88e3 2024-11-26T14:16:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/7d3ef766c64644998691bbe31ab00f33 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33 2024-11-26T14:16:38,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33, entries=200, sequenceid=131, filesize=38.7 K 2024-11-26T14:16:38,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/43e86e6225a64d418f2aff5f025c32fb as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb 2024-11-26T14:16:38,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T14:16:38,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/fbb6026c0516492b886d518c23ac88e3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3 2024-11-26T14:16:38,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T14:16:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 861ms, sequenceid=131, compaction requested=true 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:38,311 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:38,311 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:38,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:38,312 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:38,312 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:38,312 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/A is initiating minor compaction (all files) 2024-11-26T14:16:38,312 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/B is initiating minor compaction (all files) 2024-11-26T14:16:38,312 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/A in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:38,313 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=99.4 K 2024-11-26T14:16:38,313 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:38,313 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33] 2024-11-26T14:16:38,313 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/B in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:38,313 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/a89c2b32e6c741c68ee5747aa708f9f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.4 K 2024-11-26T14:16:38,313 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbeb4de6c81f40c499df6bdc6e560c3b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:38,313 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a89c2b32e6c741c68ee5747aa708f9f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:38,313 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933d1209b28d45aba9dde8118f1840ef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732630589068 2024-11-26T14:16:38,314 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting fb220024389a4e20b2bc790d059bc9b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732630589068 2024-11-26T14:16:38,314 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3ef766c64644998691bbe31ab00f33, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:38,314 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e86e6225a64d418f2aff5f025c32fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:38,320 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:38,322 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#B#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:38,323 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/5a0b833236c44653870fdc261c73a6b6 is 50, key is test_row_0/B:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:38,331 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264c9a48dfa3264f579517105967f0f625_003c34ee972d42fbe3c2e22bf0ce4d43 store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:38,333 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264c9a48dfa3264f579517105967f0f625_003c34ee972d42fbe3c2e22bf0ce4d43, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:38,333 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264c9a48dfa3264f579517105967f0f625_003c34ee972d42fbe3c2e22bf0ce4d43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:38,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742197_1373 (size=12359) 2024-11-26T14:16:38,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742198_1374 (size=4469) 2024-11-26T14:16:38,339 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/5a0b833236c44653870fdc261c73a6b6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a0b833236c44653870fdc261c73a6b6 2024-11-26T14:16:38,343 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/B of 003c34ee972d42fbe3c2e22bf0ce4d43 into 5a0b833236c44653870fdc261c73a6b6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:38,343 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:38,343 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/B, priority=13, startTime=1732630598311; duration=0sec 2024-11-26T14:16:38,343 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:38,343 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:B 2024-11-26T14:16:38,343 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:38,344 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:38,344 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/C is initiating minor compaction (all files) 2024-11-26T14:16:38,344 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/C in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:38,345 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/a25bdf17e02a4cf7942ecf66ec2eb58f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.4 K 2024-11-26T14:16:38,345 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a25bdf17e02a4cf7942ecf66ec2eb58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630587786 2024-11-26T14:16:38,348 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c019235c448645dba15060916d507411, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732630589068 2024-11-26T14:16:38,348 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb6026c0516492b886d518c23ac88e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:38,355 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#C#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:38,355 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/6e4d115bd8974acf928c14aca00dcaee is 50, key is test_row_0/C:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:38,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742199_1375 (size=12359) 2024-11-26T14:16:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T14:16:38,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:38,418 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:38,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262df29d4bf605409e94d078e1b48a9087_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630597632/Put/seqid=0 2024-11-26T14:16:38,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742200_1376 (size=12304) 2024-11-26T14:16:38,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:38,434 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262df29d4bf605409e94d078e1b48a9087_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262df29d4bf605409e94d078e1b48a9087_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:38,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c388841b8ba24cd38d094495aea1c769, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:38,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c388841b8ba24cd38d094495aea1c769 is 175, key is test_row_0/A:col10/1732630597632/Put/seqid=0 2024-11-26T14:16:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742201_1377 (size=31105) 2024-11-26T14:16:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T14:16:38,739 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:38,739 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b043f68e2d3143278b32c782b3ad95c6 is 175, key is test_row_0/A:col10/1732630597216/Put/seqid=0 2024-11-26T14:16:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742202_1378 (size=31313) 2024-11-26T14:16:38,764 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/6e4d115bd8974acf928c14aca00dcaee as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6e4d115bd8974acf928c14aca00dcaee 2024-11-26T14:16:38,768 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/C of 003c34ee972d42fbe3c2e22bf0ce4d43 into 6e4d115bd8974acf928c14aca00dcaee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:38,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:38,768 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/C, priority=13, startTime=1732630598311; duration=0sec 2024-11-26T14:16:38,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:38,768 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:C 2024-11-26T14:16:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:38,839 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c388841b8ba24cd38d094495aea1c769 2024-11-26T14:16:38,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/b1393f689a4644869eeb4a68825f249d is 50, key is test_row_0/B:col10/1732630597632/Put/seqid=0 2024-11-26T14:16:38,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742203_1379 (size=12151) 2024-11-26T14:16:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630658873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T14:16:38,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630658983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:39,146 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b043f68e2d3143278b32c782b3ad95c6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6 2024-11-26T14:16:39,150 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/A of 003c34ee972d42fbe3c2e22bf0ce4d43 into b043f68e2d3143278b32c782b3ad95c6(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:39,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:39,150 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/A, priority=13, startTime=1732630598311; duration=0sec 2024-11-26T14:16:39,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:39,150 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:A 2024-11-26T14:16:39,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630659193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:39,248 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/b1393f689a4644869eeb4a68825f249d 2024-11-26T14:16:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/70423a3fd34c44a780e12ad5955a292a is 50, key is test_row_0/C:col10/1732630597632/Put/seqid=0 2024-11-26T14:16:39,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742204_1380 (size=12151) 2024-11-26T14:16:39,263 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/70423a3fd34c44a780e12ad5955a292a 2024-11-26T14:16:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c388841b8ba24cd38d094495aea1c769 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769 2024-11-26T14:16:39,271 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769, entries=150, sequenceid=152, filesize=30.4 K 2024-11-26T14:16:39,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/b1393f689a4644869eeb4a68825f249d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d 2024-11-26T14:16:39,276 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d, entries=150, sequenceid=152, filesize=11.9 K 2024-11-26T14:16:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/70423a3fd34c44a780e12ad5955a292a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a 2024-11-26T14:16:39,281 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a, entries=150, sequenceid=152, filesize=11.9 K 2024-11-26T14:16:39,281 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 863ms, sequenceid=152, compaction requested=false 2024-11-26T14:16:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-26T14:16:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-26T14:16:39,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-26T14:16:39,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-26T14:16:39,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.0210 sec 2024-11-26T14:16:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T14:16:39,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:39,404 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-26T14:16:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T14:16:39,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:39,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:39,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:39,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T14:16:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268544e180aae24658ad4cb3f3621b7419_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:39,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742205_1381 (size=14794) 2024-11-26T14:16:39,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:39,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268544e180aae24658ad4cb3f3621b7419_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268544e180aae24658ad4cb3f3621b7419_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:39,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/534c0ba047f84e99a64dd2a2005eea50, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:39,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/534c0ba047f84e99a64dd2a2005eea50 is 175, key is test_row_0/A:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:39,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742206_1382 (size=39749) 2024-11-26T14:16:39,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/534c0ba047f84e99a64dd2a2005eea50 2024-11-26T14:16:39,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/f4182cdeccfb4523bb30fc5d8f3e538c is 50, key is test_row_0/B:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:39,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742207_1383 (size=12151) 2024-11-26T14:16:39,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T14:16:39,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:39,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630659625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:39,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T14:16:39,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:39,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T14:16:39,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630659743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:39,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T14:16:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:39,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/f4182cdeccfb4523bb30fc5d8f3e538c 2024-11-26T14:16:39,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d45c9ce4e1db4994b0cc85662ac51af8 is 50, key is test_row_0/C:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:39,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742208_1384 (size=12151) 2024-11-26T14:16:39,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d45c9ce4e1db4994b0cc85662ac51af8 2024-11-26T14:16:39,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630659954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:39,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:39,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/534c0ba047f84e99a64dd2a2005eea50 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50 2024-11-26T14:16:39,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50, entries=200, sequenceid=169, filesize=38.8 K 2024-11-26T14:16:39,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/f4182cdeccfb4523bb30fc5d8f3e538c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c 2024-11-26T14:16:39,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T14:16:39,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/d45c9ce4e1db4994b0cc85662ac51af8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8 2024-11-26T14:16:39,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T14:16:39,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 472ms, sequenceid=169, compaction requested=true 2024-11-26T14:16:39,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:39,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:39,981 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:39,981 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/A is initiating minor compaction (all files) 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/B is initiating minor compaction (all files) 2024-11-26T14:16:39,982 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/A in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,982 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/B in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,982 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a0b833236c44653870fdc261c73a6b6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.8 K 2024-11-26T14:16:39,982 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=99.8 K 2024-11-26T14:16:39,982 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50] 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0b833236c44653870fdc261c73a6b6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b043f68e2d3143278b32c782b3ad95c6, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b1393f689a4644869eeb4a68825f249d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732630597596 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c388841b8ba24cd38d094495aea1c769, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732630597596 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f4182cdeccfb4523bb30fc5d8f3e538c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:39,982 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534c0ba047f84e99a64dd2a2005eea50, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:39,988 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:39,989 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/65c0a1f7415847b8a9622ee052428300 is 50, key is test_row_0/B:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:39,989 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:39,992 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126ced2500699d94033970826f849086ecf_003c34ee972d42fbe3c2e22bf0ce4d43 store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:39,995 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126ced2500699d94033970826f849086ecf_003c34ee972d42fbe3c2e22bf0ce4d43, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:39,995 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ced2500699d94033970826f849086ecf_003c34ee972d42fbe3c2e22bf0ce4d43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:40,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742209_1385 (size=12561) 2024-11-26T14:16:40,011 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/65c0a1f7415847b8a9622ee052428300 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/65c0a1f7415847b8a9622ee052428300 2024-11-26T14:16:40,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-26T14:16:40,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,017 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T14:16:40,018 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/B of 003c34ee972d42fbe3c2e22bf0ce4d43 into 65c0a1f7415847b8a9622ee052428300(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:40,018 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:40,018 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/B, priority=13, startTime=1732630599981; duration=0sec 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:40,018 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:40,018 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:B 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:40,018 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,019 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:40,019 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/C is initiating minor compaction (all files) 2024-11-26T14:16:40,019 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/C in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,019 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6e4d115bd8974acf928c14aca00dcaee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=35.8 K 2024-11-26T14:16:40,019 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4d115bd8974acf928c14aca00dcaee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732630597216 2024-11-26T14:16:40,019 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 70423a3fd34c44a780e12ad5955a292a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732630597596 2024-11-26T14:16:40,020 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d45c9ce4e1db4994b0cc85662ac51af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:40,029 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:40,029 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/2dbac418864843c2b23eea8bc2ac603e is 50, key is test_row_0/C:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:40,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666405183d36f4f0db4de0c9d784b4557_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630599592/Put/seqid=0 2024-11-26T14:16:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T14:16:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742211_1387 (size=12561) 2024-11-26T14:16:40,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742210_1386 (size=4469) 2024-11-26T14:16:40,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742212_1388 (size=12304) 2024-11-26T14:16:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,050 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666405183d36f4f0db4de0c9d784b4557_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666405183d36f4f0db4de0c9d784b4557_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/05e607e329a7483aaba0fd361e3eb92a, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/05e607e329a7483aaba0fd361e3eb92a is 175, key is test_row_0/A:col10/1732630599592/Put/seqid=0 2024-11-26T14:16:40,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742213_1389 (size=31105) 2024-11-26T14:16:40,060 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/05e607e329a7483aaba0fd361e3eb92a 2024-11-26T14:16:40,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/adf0257db6b0491ea2c0441128076749 is 50, key is test_row_0/B:col10/1732630599592/Put/seqid=0 2024-11-26T14:16:40,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742214_1390 (size=12151) 2024-11-26T14:16:40,078 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/adf0257db6b0491ea2c0441128076749 2024-11-26T14:16:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/1ddcddcddc774b3daa62fbc607d8bdaa is 50, key is test_row_0/C:col10/1732630599592/Put/seqid=0 2024-11-26T14:16:40,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742215_1391 (size=12151) 2024-11-26T14:16:40,101 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/1ddcddcddc774b3daa62fbc607d8bdaa 2024-11-26T14:16:40,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/05e607e329a7483aaba0fd361e3eb92a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a 2024-11-26T14:16:40,115 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a, entries=150, sequenceid=187, filesize=30.4 K 2024-11-26T14:16:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/adf0257db6b0491ea2c0441128076749 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749 2024-11-26T14:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,119 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749, entries=150, sequenceid=187, filesize=11.9 K 2024-11-26T14:16:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/1ddcddcddc774b3daa62fbc607d8bdaa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa, entries=150, sequenceid=187, filesize=11.9 K 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 108ms, sequenceid=187, compaction requested=false 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:40,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-26T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-26T14:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 719 msec 2024-11-26T14:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 725 msec 2024-11-26T14:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:40,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f6688c282c0b4928af84ce2f18539c2d_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742216_1392 (size=19774) 2024-11-26T14:16:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,438 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/2dbac418864843c2b23eea8bc2ac603e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/2dbac418864843c2b23eea8bc2ac603e 2024-11-26T14:16:40,440 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:40,441 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/a0b37356a3f94d84959d925550a5e2c8 is 175, key is test_row_0/A:col10/1732630598861/Put/seqid=0 2024-11-26T14:16:40,442 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/C of 003c34ee972d42fbe3c2e22bf0ce4d43 into 2dbac418864843c2b23eea8bc2ac603e(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:40,442 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:40,443 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/C, priority=13, startTime=1732630599981; duration=0sec 2024-11-26T14:16:40,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:40,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:C 2024-11-26T14:16:40,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742217_1393 (size=31515) 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630600414, max timestamp=1732630600446Num uniques 859; 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630600414, max timestamp=1732630600446Num uniques 859; 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630600414, max timestamp=1732630600446Num uniques 859; 2024-11-26T14:16:40,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:16:40,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630660472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630660480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630660481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630660487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630660491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-26T14:16:40,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:40,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:16:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-26T14:16:40,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:16:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:40,546 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:16:40,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:16:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630660593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630660593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630660603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630660604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630660604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:40,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,789 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:40,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f6688c282c0b4928af84ce2f18539c2d_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f6688c282c0b4928af84ce2f18539c2d_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:40,794 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c084ce2b72a04276b6bf6e826f1f6f81, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:40,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c084ce2b72a04276b6bf6e826f1f6f81 is 175, key is test_row_0/A:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:40,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742218_1394 (size=57033) 2024-11-26T14:16:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630660813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630660813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630660823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630660823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630660823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:40,849 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/a0b37356a3f94d84959d925550a5e2c8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8 2024-11-26T14:16:40,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:40,852 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/A of 003c34ee972d42fbe3c2e22bf0ce4d43 into a0b37356a3f94d84959d925550a5e2c8(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,852 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:40,852 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/A, priority=13, startTime=1732630599981; duration=0sec 2024-11-26T14:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:40,852 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:40,852 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:A 2024-11-26T14:16:40,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:41,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630661133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630661133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630661143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630661143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630661143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:41,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c084ce2b72a04276b6bf6e826f1f6f81 2024-11-26T14:16:41,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/271a9462264a415495a1a582059b208f is 50, key is test_row_0/B:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:41,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742219_1395 (size=12151) 2024-11-26T14:16:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/271a9462264a415495a1a582059b208f 2024-11-26T14:16:41,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/20507cd75a1f4d09b74c4574683593eb is 50, key is test_row_0/C:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:41,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742220_1396 (size=12151) 2024-11-26T14:16:41,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55434 deadline: 1732630661653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55442 deadline: 1732630661653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55374 deadline: 1732630661663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55422 deadline: 1732630661663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55440 deadline: 1732630661663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:16:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:16:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:41,740 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-26T14:16:41,740 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-26T14:16:41,740 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:41,740 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:41,740 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,740 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,740 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:41,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:41,740 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,743 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-26T14:16:41,743 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:41,743 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,743 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:41,753 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-26T14:16:41,753 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-26T14:16:41,753 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:41,753 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:41,753 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,753 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,754 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,754 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:41,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:41,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:41,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. as already flushing 2024-11-26T14:16:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:41,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:16:42,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/20507cd75a1f4d09b74c4574683593eb 2024-11-26T14:16:42,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/c084ce2b72a04276b6bf6e826f1f6f81 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81 2024-11-26T14:16:42,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81, entries=300, sequenceid=197, filesize=55.7 K 2024-11-26T14:16:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/271a9462264a415495a1a582059b208f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f 2024-11-26T14:16:42,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T14:16:42,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/20507cd75a1f4d09b74c4574683593eb as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb 2024-11-26T14:16:42,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T14:16:42,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1665ms, sequenceid=197, compaction requested=true 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:42,030 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 003c34ee972d42fbe3c2e22bf0ce4d43:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:16:42,030 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:42,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:42,031 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:42,031 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:42,031 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/A is initiating minor compaction (all files) 2024-11-26T14:16:42,031 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/B is initiating minor compaction (all files) 2024-11-26T14:16:42,031 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/A in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,031 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/B in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,031 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=116.8 K 2024-11-26T14:16:42,031 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/65c0a1f7415847b8a9622ee052428300, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=36.0 K 2024-11-26T14:16:42,031 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,031 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81] 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c0a1f7415847b8a9622ee052428300, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0b37356a3f94d84959d925550a5e2c8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting adf0257db6b0491ea2c0441128076749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732630599582 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05e607e329a7483aaba0fd361e3eb92a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732630599582 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 271a9462264a415495a1a582059b208f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630600360 2024-11-26T14:16:42,032 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c084ce2b72a04276b6bf6e826f1f6f81, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630600346 2024-11-26T14:16:42,036 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:42,038 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:42,039 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/8706671177d6418ba48d54fda3685c24 is 50, key is test_row_0/B:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:42,041 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126b9c5bf8c59d048cd8b62e92c451f41f0_003c34ee972d42fbe3c2e22bf0ce4d43 store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742221_1397 (size=12663) 2024-11-26T14:16:42,049 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126b9c5bf8c59d048cd8b62e92c451f41f0_003c34ee972d42fbe3c2e22bf0ce4d43, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:42,049 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b9c5bf8c59d048cd8b62e92c451f41f0_003c34ee972d42fbe3c2e22bf0ce4d43 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:42,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742222_1398 (size=4469) 2024-11-26T14:16:42,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-26T14:16:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,087 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:16:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f442817a81454e41b40c7638e7ad0410_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630600485/Put/seqid=0 2024-11-26T14:16:42,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742223_1399 (size=12304) 2024-11-26T14:16:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:42,107 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f442817a81454e41b40c7638e7ad0410_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f442817a81454e41b40c7638e7ad0410_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/80ec2a4ed8b64f56b39afcb4fd327b96, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:42,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/80ec2a4ed8b64f56b39afcb4fd327b96 is 175, key is test_row_0/A:col10/1732630600485/Put/seqid=0 2024-11-26T14:16:42,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742224_1400 (size=31105) 2024-11-26T14:16:42,121 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/80ec2a4ed8b64f56b39afcb4fd327b96 2024-11-26T14:16:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/10a769c979bc4d1d89fe8559e5be330d is 50, key is test_row_0/B:col10/1732630600485/Put/seqid=0 2024-11-26T14:16:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742225_1401 (size=12151) 2024-11-26T14:16:42,141 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/10a769c979bc4d1d89fe8559e5be330d 2024-11-26T14:16:42,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/edda2b27d7c94acbae3041964bafec9d is 50, key is test_row_0/C:col10/1732630600485/Put/seqid=0 2024-11-26T14:16:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742226_1402 (size=12151) 2024-11-26T14:16:42,448 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/8706671177d6418ba48d54fda3685c24 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/8706671177d6418ba48d54fda3685c24 2024-11-26T14:16:42,454 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/B of 003c34ee972d42fbe3c2e22bf0ce4d43 into 8706671177d6418ba48d54fda3685c24(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:42,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:42,455 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/B, priority=13, startTime=1732630602030; duration=0sec 2024-11-26T14:16:42,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:16:42,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:B 2024-11-26T14:16:42,455 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#A#compaction#332 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:42,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:16:42,456 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/6bf886c5e3e34beb9693dba134933381 is 175, key is test_row_0/A:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:42,456 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:16:42,456 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 003c34ee972d42fbe3c2e22bf0ce4d43/C is initiating minor compaction (all files) 2024-11-26T14:16:42,456 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 003c34ee972d42fbe3c2e22bf0ce4d43/C in TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,456 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/2dbac418864843c2b23eea8bc2ac603e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp, totalSize=36.0 K 2024-11-26T14:16:42,457 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dbac418864843c2b23eea8bc2ac603e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732630598816 2024-11-26T14:16:42,458 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ddcddcddc774b3daa62fbc607d8bdaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732630599582 2024-11-26T14:16:42,458 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 20507cd75a1f4d09b74c4574683593eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630600360 2024-11-26T14:16:42,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742227_1403 (size=31617) 2024-11-26T14:16:42,465 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 003c34ee972d42fbe3c2e22bf0ce4d43#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:16:42,465 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/19231522ec5942158ade41c119acf9d7 is 50, key is test_row_0/C:col10/1732630600360/Put/seqid=0 2024-11-26T14:16:42,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742228_1404 (size=12663) 2024-11-26T14:16:42,554 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/edda2b27d7c94acbae3041964bafec9d 2024-11-26T14:16:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/80ec2a4ed8b64f56b39afcb4fd327b96 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/80ec2a4ed8b64f56b39afcb4fd327b96 2024-11-26T14:16:42,567 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/80ec2a4ed8b64f56b39afcb4fd327b96, entries=150, sequenceid=234, filesize=30.4 K 2024-11-26T14:16:42,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/10a769c979bc4d1d89fe8559e5be330d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/10a769c979bc4d1d89fe8559e5be330d 2024-11-26T14:16:42,572 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/10a769c979bc4d1d89fe8559e5be330d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T14:16:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/edda2b27d7c94acbae3041964bafec9d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/edda2b27d7c94acbae3041964bafec9d 2024-11-26T14:16:42,576 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/edda2b27d7c94acbae3041964bafec9d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-26T14:16:42,577 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 490ms, sequenceid=234, compaction requested=false 2024-11-26T14:16:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-26T14:16:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-26T14:16:42,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-26T14:16:42,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-11-26T14:16:42,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.0370 sec 2024-11-26T14:16:42,674 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-26T14:16:42,674 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:42,675 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,675 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-26T14:16:42,675 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:42,675 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-26T14:16:42,675 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:42,675 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,675 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,684 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-26T14:16:42,684 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-26T14:16:42,684 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:42,684 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:16:42,684 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,684 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,684 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,684 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-26T14:16:42,693 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 934 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2797 rows 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 928 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2773 rows 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 930 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2783 rows 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 924 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2768 rows 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 927 2024-11-26T14:16:42,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2773 rows 2024-11-26T14:16:42,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:16:42,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:16:42,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:16:42,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:16:42,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:16:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:42,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630602697"}]},"ts":"1732630602697"} 2024-11-26T14:16:42,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:16:42,698 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:16:42,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:16:42,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, UNASSIGN}] 2024-11-26T14:16:42,701 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, UNASSIGN 2024-11-26T14:16:42,701 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=95 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:42,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, UNASSIGN because future has completed 2024-11-26T14:16:42,702 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:16:42,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:42,855 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(122): Close 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:42,855 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:16:42,856 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1722): Closing 003c34ee972d42fbe3c2e22bf0ce4d43, disabling compactions & flushes 2024-11-26T14:16:42,856 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,872 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/6bf886c5e3e34beb9693dba134933381 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/6bf886c5e3e34beb9693dba134933381 2024-11-26T14:16:42,875 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/19231522ec5942158ade41c119acf9d7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/19231522ec5942158ade41c119acf9d7 2024-11-26T14:16:42,877 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/A of 003c34ee972d42fbe3c2e22bf0ce4d43 into 6bf886c5e3e34beb9693dba134933381(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:42,877 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:42,877 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/A, priority=13, startTime=1732630602030; duration=0sec 2024-11-26T14:16:42,877 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:42,877 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:A 2024-11-26T14:16:42,880 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 003c34ee972d42fbe3c2e22bf0ce4d43/C of 003c34ee972d42fbe3c2e22bf0ce4d43 into 19231522ec5942158ade41c119acf9d7(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:16:42,880 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 003c34ee972d42fbe3c2e22bf0ce4d43: 2024-11-26T14:16:42,880 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,880 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., storeName=003c34ee972d42fbe3c2e22bf0ce4d43/C, priority=13, startTime=1732630602030; duration=0sec 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,880 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:16:42,880 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 003c34ee972d42fbe3c2e22bf0ce4d43:C 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. after waiting 0 ms 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:42,880 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(2902): Flushing 003c34ee972d42fbe3c2e22bf0ce4d43 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=A 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=B 2024-11-26T14:16:42,880 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:42,881 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 003c34ee972d42fbe3c2e22bf0ce4d43, store=C 2024-11-26T14:16:42,881 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:16:42,886 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126363d5c2634f14b2e8bd27155190c1842_003c34ee972d42fbe3c2e22bf0ce4d43 is 50, key is test_row_0/A:col10/1732630602683/Put/seqid=0 2024-11-26T14:16:42,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742229_1405 (size=9814) 2024-11-26T14:16:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:43,290 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:43,299 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126363d5c2634f14b2e8bd27155190c1842_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126363d5c2634f14b2e8bd27155190c1842_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:43,299 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b43f7f29536b4977b6a804a1cc8b62a5, store: [table=TestAcidGuarantees family=A region=003c34ee972d42fbe3c2e22bf0ce4d43] 2024-11-26T14:16:43,300 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b43f7f29536b4977b6a804a1cc8b62a5 is 175, key is test_row_0/A:col10/1732630602683/Put/seqid=0 2024-11-26T14:16:43,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742230_1406 (size=22461) 2024-11-26T14:16:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:43,704 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b43f7f29536b4977b6a804a1cc8b62a5 2024-11-26T14:16:43,710 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/badedc18de1b42328fbf5fc77bd44479 is 50, key is test_row_0/B:col10/1732630602683/Put/seqid=0 2024-11-26T14:16:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742231_1407 (size=9757) 2024-11-26T14:16:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:44,114 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/badedc18de1b42328fbf5fc77bd44479 2024-11-26T14:16:44,121 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/3102c96b532c4481845d90924cc465be is 50, key is test_row_0/C:col10/1732630602683/Put/seqid=0 2024-11-26T14:16:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742232_1408 (size=9757) 2024-11-26T14:16:44,124 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/3102c96b532c4481845d90924cc465be 2024-11-26T14:16:44,129 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/A/b43f7f29536b4977b6a804a1cc8b62a5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b43f7f29536b4977b6a804a1cc8b62a5 2024-11-26T14:16:44,133 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b43f7f29536b4977b6a804a1cc8b62a5, entries=100, sequenceid=245, filesize=21.9 K 2024-11-26T14:16:44,133 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/B/badedc18de1b42328fbf5fc77bd44479 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/badedc18de1b42328fbf5fc77bd44479 2024-11-26T14:16:44,138 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/badedc18de1b42328fbf5fc77bd44479, entries=100, sequenceid=245, filesize=9.5 K 2024-11-26T14:16:44,139 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/.tmp/C/3102c96b532c4481845d90924cc465be as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/3102c96b532c4481845d90924cc465be 2024-11-26T14:16:44,143 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/3102c96b532c4481845d90924cc465be, entries=100, sequenceid=245, filesize=9.5 K 2024-11-26T14:16:44,144 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1264ms, sequenceid=245, compaction requested=true 2024-11-26T14:16:44,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81] to archive 2024-11-26T14:16:44,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:44,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/30866d159e8d4e96b96679324670e004 2024-11-26T14:16:44,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7618b758b01242188edb5e64db94f78d 2024-11-26T14:16:44,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/d4b08dbb4ae84f3498415d3a850fc4ac 2024-11-26T14:16:44,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b185a1da02bd4773948a5a48ce55ed24 2024-11-26T14:16:44,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/930de1336b004020aed98830f207e548 2024-11-26T14:16:44,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/0d4b61b2483041f9a7ed35f3c3ba0cbf 2024-11-26T14:16:44,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/dbeb4de6c81f40c499df6bdc6e560c3b 2024-11-26T14:16:44,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/933d1209b28d45aba9dde8118f1840ef 2024-11-26T14:16:44,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/7d3ef766c64644998691bbe31ab00f33 2024-11-26T14:16:44,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b043f68e2d3143278b32c782b3ad95c6 2024-11-26T14:16:44,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c388841b8ba24cd38d094495aea1c769 2024-11-26T14:16:44,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/534c0ba047f84e99a64dd2a2005eea50 2024-11-26T14:16:44,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/a0b37356a3f94d84959d925550a5e2c8 2024-11-26T14:16:44,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/05e607e329a7483aaba0fd361e3eb92a 2024-11-26T14:16:44,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/c084ce2b72a04276b6bf6e826f1f6f81 2024-11-26T14:16:44,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/3df3a9bb58f543038b04100393a9f60f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/a89c2b32e6c741c68ee5747aa708f9f7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a0b833236c44653870fdc261c73a6b6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/65c0a1f7415847b8a9622ee052428300, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f] to archive 2024-11-26T14:16:44,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:44,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/cc30610b82dd42919a84051574002684 2024-11-26T14:16:44,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/7aa766511f3145639026c8cc42372d09 2024-11-26T14:16:44,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/3df3a9bb58f543038b04100393a9f60f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/3df3a9bb58f543038b04100393a9f60f 2024-11-26T14:16:44,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/d5985fcde073467890305a3df3e93eb1 2024-11-26T14:16:44,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/ab039399f80240a28db0133254fb506e 2024-11-26T14:16:44,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/a89c2b32e6c741c68ee5747aa708f9f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/a89c2b32e6c741c68ee5747aa708f9f7 2024-11-26T14:16:44,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a3be68819f94e6eb9dbd1ce55cfa72a 2024-11-26T14:16:44,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/fb220024389a4e20b2bc790d059bc9b6 2024-11-26T14:16:44,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a0b833236c44653870fdc261c73a6b6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/5a0b833236c44653870fdc261c73a6b6 2024-11-26T14:16:44,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/43e86e6225a64d418f2aff5f025c32fb 2024-11-26T14:16:44,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/b1393f689a4644869eeb4a68825f249d 2024-11-26T14:16:44,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/65c0a1f7415847b8a9622ee052428300 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/65c0a1f7415847b8a9622ee052428300 2024-11-26T14:16:44,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/f4182cdeccfb4523bb30fc5d8f3e538c 2024-11-26T14:16:44,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/adf0257db6b0491ea2c0441128076749 2024-11-26T14:16:44,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/271a9462264a415495a1a582059b208f 2024-11-26T14:16:44,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/dcb7b1fa2c1e4660a002f7154b714384, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/a25bdf17e02a4cf7942ecf66ec2eb58f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6e4d115bd8974acf928c14aca00dcaee, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/2dbac418864843c2b23eea8bc2ac603e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb] to archive 2024-11-26T14:16:44,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:16:44,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/aa82f34b6fca46fbb28376ee73892eb3 2024-11-26T14:16:44,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/f2c2cd753d694d2381a0966ad933a30f 2024-11-26T14:16:44,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/dcb7b1fa2c1e4660a002f7154b714384 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/dcb7b1fa2c1e4660a002f7154b714384 2024-11-26T14:16:44,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6148e8d8ca3c499aa0e1b02d94c25d4b 2024-11-26T14:16:44,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/5b90da77e83844e0b9cd43d8306684b7 2024-11-26T14:16:44,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/a25bdf17e02a4cf7942ecf66ec2eb58f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/a25bdf17e02a4cf7942ecf66ec2eb58f 2024-11-26T14:16:44,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d1116e9dfce54a469e4a194c27f851ba 2024-11-26T14:16:44,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/c019235c448645dba15060916d507411 2024-11-26T14:16:44,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6e4d115bd8974acf928c14aca00dcaee to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/6e4d115bd8974acf928c14aca00dcaee 2024-11-26T14:16:44,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/fbb6026c0516492b886d518c23ac88e3 2024-11-26T14:16:44,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/70423a3fd34c44a780e12ad5955a292a 2024-11-26T14:16:44,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/2dbac418864843c2b23eea8bc2ac603e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/2dbac418864843c2b23eea8bc2ac603e 2024-11-26T14:16:44,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/d45c9ce4e1db4994b0cc85662ac51af8 2024-11-26T14:16:44,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/1ddcddcddc774b3daa62fbc607d8bdaa 2024-11-26T14:16:44,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/20507cd75a1f4d09b74c4574683593eb 2024-11-26T14:16:44,195 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits/248.seqid, newMaxSeqId=248, maxSeqId=4 2024-11-26T14:16:44,196 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. 2024-11-26T14:16:44,196 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] regionserver.HRegion(1676): Region close journal for 003c34ee972d42fbe3c2e22bf0ce4d43: Waiting for close lock at 1732630602856Running coprocessor pre-close hooks at 1732630602856Disabling compacts and flushes for region at 1732630602856Disabling writes for close at 1732630602880 (+24 ms)Obtaining lock to block concurrent updates at 1732630602880Preparing flush snapshotting stores in 003c34ee972d42fbe3c2e22bf0ce4d43 at 1732630602880Finished memstore snapshotting TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732630602881 (+1 ms)Flushing stores of TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43. at 1732630602881Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/A: creating writer at 1732630602881Flushing mob file 003c34ee972d42fbe3c2e22bf0ce4d43/A: appending metadata at 1732630602885 (+4 ms)Flushing mob file 003c34ee972d42fbe3c2e22bf0ce4d43/A: closing flushed file at 1732630602885Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/A: appending metadata at 1732630603300 (+415 ms)Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/A: closing flushed file at 1732630603300Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/B: creating writer at 1732630603708 (+408 ms)Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/B: appending metadata at 1732630603710 (+2 ms)Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/B: closing flushed file at 1732630603710Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/C: creating writer at 1732630604117 (+407 ms)Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/C: appending metadata at 1732630604120 (+3 ms)Flushing 003c34ee972d42fbe3c2e22bf0ce4d43/C: closing flushed file at 1732630604120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498c442: reopening flushed file at 1732630604128 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69b57ba6: reopening flushed file at 1732630604133 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35e180c9: reopening flushed file at 1732630604138 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 003c34ee972d42fbe3c2e22bf0ce4d43 in 1264ms, sequenceid=245, compaction requested=true at 1732630604144 (+6 ms)Writing region close event to WAL at 1732630604193 (+49 ms)Running coprocessor post-close hooks at 1732630604196 (+3 ms)Closed at 1732630604196 2024-11-26T14:16:44,197 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=96}] handler.UnassignRegionHandler(157): Closed 003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=95 updating hbase:meta row=003c34ee972d42fbe3c2e22bf0ce4d43, regionState=CLOSED 2024-11-26T14:16:44,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:44,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T14:16:44,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; CloseRegionProcedure 003c34ee972d42fbe3c2e22bf0ce4d43, server=19945b63c2b4,40025,1732630490995 in 1.4970 sec 2024-11-26T14:16:44,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-26T14:16:44,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=003c34ee972d42fbe3c2e22bf0ce4d43, UNASSIGN in 1.5000 sec 2024-11-26T14:16:44,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T14:16:44,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5030 sec 2024-11-26T14:16:44,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630604205"}]},"ts":"1732630604205"} 2024-11-26T14:16:44,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:16:44,206 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:16:44,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5130 sec 2024-11-26T14:16:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T14:16:44,844 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:44,844 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:16:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,846 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=97, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,847 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=97, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T14:16:44,850 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,852 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits] 2024-11-26T14:16:44,859 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/6bf886c5e3e34beb9693dba134933381 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/6bf886c5e3e34beb9693dba134933381 2024-11-26T14:16:44,860 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/80ec2a4ed8b64f56b39afcb4fd327b96 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/80ec2a4ed8b64f56b39afcb4fd327b96 2024-11-26T14:16:44,862 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b43f7f29536b4977b6a804a1cc8b62a5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/A/b43f7f29536b4977b6a804a1cc8b62a5 2024-11-26T14:16:44,867 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/10a769c979bc4d1d89fe8559e5be330d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/10a769c979bc4d1d89fe8559e5be330d 2024-11-26T14:16:44,869 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/8706671177d6418ba48d54fda3685c24 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/8706671177d6418ba48d54fda3685c24 2024-11-26T14:16:44,869 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/badedc18de1b42328fbf5fc77bd44479 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/B/badedc18de1b42328fbf5fc77bd44479 2024-11-26T14:16:44,872 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/19231522ec5942158ade41c119acf9d7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/19231522ec5942158ade41c119acf9d7 2024-11-26T14:16:44,873 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/3102c96b532c4481845d90924cc465be to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/3102c96b532c4481845d90924cc465be 2024-11-26T14:16:44,874 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/edda2b27d7c94acbae3041964bafec9d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/C/edda2b27d7c94acbae3041964bafec9d 2024-11-26T14:16:44,878 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits/248.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43/recovered.edits/248.seqid 2024-11-26T14:16:44,879 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,879 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:16:44,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:16:44,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T14:16:44,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094070aa7d694edfbc6b5f3e6804c6d4_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094070aa7d694edfbc6b5f3e6804c6d4_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262df29d4bf605409e94d078e1b48a9087_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262df29d4bf605409e94d078e1b48a9087_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126363d5c2634f14b2e8bd27155190c1842_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126363d5c2634f14b2e8bd27155190c1842_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126457fc7b00cb44764bdc4c96c575920a7_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126457fc7b00cb44764bdc4c96c575920a7_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112663e53a7d9d554b2796c4a03894426a89_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112663e53a7d9d554b2796c4a03894426a89_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666405183d36f4f0db4de0c9d784b4557_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666405183d36f4f0db4de0c9d784b4557_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268544e180aae24658ad4cb3f3621b7419_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268544e180aae24658ad4cb3f3621b7419_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3158c10b2de4c96a0a9635f4885dcb7_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3158c10b2de4c96a0a9635f4885dcb7_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8f22a2a01674893b0c3b85089a1811a_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8f22a2a01674893b0c3b85089a1811a_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,893 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec9e5385c14744bf9e4b6b70aa9d15cc_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ec9e5385c14744bf9e4b6b70aa9d15cc_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,894 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f3be4b30308b490c8b64e544e5693141_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f3be4b30308b490c8b64e544e5693141_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,895 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f442817a81454e41b40c7638e7ad0410_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f442817a81454e41b40c7638e7ad0410_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,896 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f6688c282c0b4928af84ce2f18539c2d_003c34ee972d42fbe3c2e22bf0ce4d43 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f6688c282c0b4928af84ce2f18539c2d_003c34ee972d42fbe3c2e22bf0ce4d43 2024-11-26T14:16:44,896 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:16:44,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=97, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,902 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:16:44,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:16:44,905 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=97, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,905 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:16:44,906 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630604905"}]},"ts":"9223372036854775807"} 2024-11-26T14:16:44,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:16:44,908 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 003c34ee972d42fbe3c2e22bf0ce4d43, NAME => 'TestAcidGuarantees,,1732630578569.003c34ee972d42fbe3c2e22bf0ce4d43.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:16:44,908 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:16:44,908 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630604908"}]},"ts":"9223372036854775807"} 2024-11-26T14:16:44,910 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:16:44,911 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=97, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-26T14:16:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T14:16:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:16:44,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:16:44,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 251) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1156 (was 1055) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5167 (was 6104) 2024-11-26T14:16:44,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1156, ProcessCount=11, AvailableMemoryMB=5167 2024-11-26T14:16:44,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:16:44,976 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:16:44,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:16:44,978 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:16:44,978 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:44,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 98 2024-11-26T14:16:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:44,979 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:16:44,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742233_1409 (size=1061) 2024-11-26T14:16:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:45,391 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:16:45,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742234_1410 (size=53) 2024-11-26T14:16:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:45,802 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:16:45,802 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c88c058694b3dba456cf23dd49f28b4, disabling compactions & flushes 2024-11-26T14:16:45,803 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:45,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:45,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. after waiting 0 ms 2024-11-26T14:16:45,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:45,803 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:45,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c88c058694b3dba456cf23dd49f28b4: Waiting for close lock at 1732630605802Disabling compacts and flushes for region at 1732630605802Disabling writes for close at 1732630605803 (+1 ms)Writing region close event to WAL at 1732630605803Closed at 1732630605803 2024-11-26T14:16:45,805 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:16:45,806 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630605805"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630605805"}]},"ts":"1732630605805"} 2024-11-26T14:16:45,809 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:16:45,810 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:16:45,811 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630605810"}]},"ts":"1732630605810"} 2024-11-26T14:16:45,813 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:16:45,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, ASSIGN}] 2024-11-26T14:16:45,814 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, ASSIGN 2024-11-26T14:16:45,814 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:16:45,965 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=4c88c058694b3dba456cf23dd49f28b4, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:45,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, ASSIGN because future has completed 2024-11-26T14:16:45,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:16:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:48,385 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:16:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:49,723 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:16:50,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T14:16:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:16:55,296 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:55,296 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:16:55,298 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,298 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:16:55,298 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,298 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,301 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,302 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:55,302 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c88c058694b3dba456cf23dd49f28b4 columnFamilyName A 2024-11-26T14:16:55,302 DEBUG [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:55,303 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(327): Store=4c88c058694b3dba456cf23dd49f28b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:55,303 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,304 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:55,304 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c88c058694b3dba456cf23dd49f28b4 columnFamilyName B 2024-11-26T14:16:55,304 DEBUG [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:55,304 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(327): Store=4c88c058694b3dba456cf23dd49f28b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:55,305 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,305 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:16:55,305 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c88c058694b3dba456cf23dd49f28b4 columnFamilyName C 2024-11-26T14:16:55,305 DEBUG [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:16:55,306 INFO [StoreOpener-4c88c058694b3dba456cf23dd49f28b4-1 {}] regionserver.HStore(327): Store=4c88c058694b3dba456cf23dd49f28b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:16:55,306 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:55,306 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,307 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,307 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,308 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,308 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,308 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:16:55,310 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,312 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:16:55,312 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 4c88c058694b3dba456cf23dd49f28b4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64705078, jitterRate=-0.03581920266151428}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:16:55,312 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:16:55,313 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 4c88c058694b3dba456cf23dd49f28b4: Running coprocessor pre-open hook at 1732630615299Writing region info on filesystem at 1732630615299Initializing all the Stores at 1732630615300 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630615300Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630615300Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630615300Cleaning up temporary data from old regions at 1732630615308 (+8 ms)Running coprocessor post-open hooks at 1732630615312 (+4 ms)Region opened successfully at 1732630615313 (+1 ms) 2024-11-26T14:16:55,313 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., pid=100, masterSystemTime=1732630606120 2024-11-26T14:16:55,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=99 updating hbase:meta row=4c88c058694b3dba456cf23dd49f28b4, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:16:55,320 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:55,320 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:16:55,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:16:55,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T14:16:55,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 in 9.3550 sec 2024-11-26T14:16:55,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-26T14:16:55,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, ASSIGN in 9.5100 sec 2024-11-26T14:16:55,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:16:55,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630615332"}]},"ts":"1732630615332"} 2024-11-26T14:16:55,334 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:16:55,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=98, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:16:55,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 10.3590 sec 2024-11-26T14:16:55,729 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:17:00,230 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T14:17:00,230 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T14:17:00,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:17:00,815 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T14:17:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-26T14:17:03,223 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:03,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,225 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd276af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,227 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49876593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,228 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:17:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:17:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38105b46 2024-11-26T14:17:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:17:03,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:17:03,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,234 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7eb686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b2ebc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ce49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebf5fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578efefb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,242 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be5b872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,244 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dd9f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362a9996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,247 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651a71e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,251 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e7fc7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,255 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435f8d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4654dbd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,263 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e1c50f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,264 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4caaba03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,270 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233f05da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6924b263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,276 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34495975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354efb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:03,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:03,286 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41911951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:03,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bed5bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:03,300 DEBUG [Thread-1866 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,301 DEBUG [Thread-1877 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,304 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,312 DEBUG [Thread-1881 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,312 DEBUG [Thread-1864 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,321 DEBUG [Thread-1879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:03,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:03,332 DEBUG [Thread-1862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,348 DEBUG [Thread-1868 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,350 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T14:17:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:03,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:03,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:03,359 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:03,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2] 2024-11-26T14:17:03,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630623343, max timestamp=1732630623365Num uniques 859; 2024-11-26T14:17:03,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:03,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630623343, max timestamp=1732630623365Num uniques 859; 2024-11-26T14:17:03,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:03,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630623343, max timestamp=1732630623365Num uniques 859; 2024-11-26T14:17:03,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:03,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/87fd4f4905d941f1a61b6b0c8d038a4a is 50, key is test_row_0/A:col10/1732630623317/Put/seqid=0 2024-11-26T14:17:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742235_1411 (size=12001) 2024-11-26T14:17:03,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/87fd4f4905d941f1a61b6b0c8d038a4a 2024-11-26T14:17:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630683419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630683421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630683425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630683426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630683440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:03,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/399c00bf05234b14bfbefbfb7d8897cf is 50, key is test_row_0/B:col10/1732630623317/Put/seqid=0 2024-11-26T14:17:03,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742236_1412 (size=12001) 2024-11-26T14:17:03,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/399c00bf05234b14bfbefbfb7d8897cf 2024-11-26T14:17:03,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630683552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630683553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630683553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630683553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/d2cd9089c67f40ccb9a7b78c8ef0b95a is 50, key is test_row_0/C:col10/1732630623317/Put/seqid=0 2024-11-26T14:17:03,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630683573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742237_1413 (size=12001) 2024-11-26T14:17:03,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:03,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630683775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630683775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630683775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630683775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630683794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:03,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:03,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:03,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:03,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/d2cd9089c67f40ccb9a7b78c8ef0b95a 2024-11-26T14:17:03,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:03,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/87fd4f4905d941f1a61b6b0c8d038a4a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a 2024-11-26T14:17:04,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:17:04,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/399c00bf05234b14bfbefbfb7d8897cf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf 2024-11-26T14:17:04,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:17:04,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/d2cd9089c67f40ccb9a7b78c8ef0b95a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a 2024-11-26T14:17:04,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-26T14:17:04,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4c88c058694b3dba456cf23dd49f28b4 in 721ms, sequenceid=11, compaction requested=false 2024-11-26T14:17:04,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:04,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:04,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:04,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/9396c9fc09c946bdb21e946a3faf341e is 50, key is test_row_0/A:col10/1732630623419/Put/seqid=0 2024-11-26T14:17:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630684096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630684104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742238_1414 (size=16681) 2024-11-26T14:17:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630684110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630684111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630684115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630684225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630684228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630684235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630684235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630684453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630684453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630684453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630684453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:04,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/9396c9fc09c946bdb21e946a3faf341e 2024-11-26T14:17:04,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/0e1993208f844513abea060c7f8c508a is 50, key is test_row_0/B:col10/1732630623419/Put/seqid=0 2024-11-26T14:17:04,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742239_1415 (size=12001) 2024-11-26T14:17:04,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630684633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630684773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630684773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630684774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630684774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:04,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:04,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/0e1993208f844513abea060c7f8c508a 2024-11-26T14:17:05,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 is 50, key is test_row_0/C:col10/1732630623419/Put/seqid=0 2024-11-26T14:17:05,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742240_1416 (size=12001) 2024-11-26T14:17:05,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630685283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630685293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630685294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:05,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630685295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:05,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:05,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 2024-11-26T14:17:05,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/9396c9fc09c946bdb21e946a3faf341e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e 2024-11-26T14:17:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:05,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e, entries=250, sequenceid=48, filesize=16.3 K 2024-11-26T14:17:05,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/0e1993208f844513abea060c7f8c508a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a 2024-11-26T14:17:05,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T14:17:05,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 2024-11-26T14:17:05,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T14:17:05,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 4c88c058694b3dba456cf23dd49f28b4 in 1450ms, sequenceid=48, compaction requested=false 2024-11-26T14:17:05,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:05,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T14:17:05,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:05,605 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:05,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:05,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/c4577a01df914a82ac4c10d398dccdc5 is 50, key is test_row_0/A:col10/1732630624103/Put/seqid=0 2024-11-26T14:17:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742241_1417 (size=12001) 2024-11-26T14:17:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:05,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:05,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630625715, max timestamp=1732630625894Num uniques 859; 2024-11-26T14:17:05,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:05,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630625715, max timestamp=1732630625894Num uniques 859; 2024-11-26T14:17:05,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630625715, max timestamp=1732630625894Num uniques 859; 2024-11-26T14:17:05,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:05,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:06,022 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/c4577a01df914a82ac4c10d398dccdc5 2024-11-26T14:17:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/87e60260a333474c8a6b2114a325f019 is 50, key is test_row_0/B:col10/1732630624103/Put/seqid=0 2024-11-26T14:17:06,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742242_1418 (size=12001) 2024-11-26T14:17:06,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630626071, max timestamp=1732630626153Num uniques 1018; 2024-11-26T14:17:06,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630626071, max timestamp=1732630626153Num uniques 1018; 2024-11-26T14:17:06,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:06,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630626071, max timestamp=1732630626153Num uniques 1018; 2024-11-26T14:17:06,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:06,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732630626280, max timestamp=1732630626331Num uniques 1027; 2024-11-26T14:17:06,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-26T14:17:06,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732630626280, max timestamp=1732630626331Num uniques 1027; 2024-11-26T14:17:06,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-26T14:17:06,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732630626280, max timestamp=1732630626331Num uniques 1027; 2024-11-26T14:17:06,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-26T14:17:06,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630686393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630626373, max timestamp=1732630626397Num uniques 1186; 2024-11-26T14:17:06,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630626373, max timestamp=1732630626397Num uniques 1186; 2024-11-26T14:17:06,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-26T14:17:06,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-26T14:17:06,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630626373, max timestamp=1732630626397Num uniques 1186; 2024-11-26T14:17:06,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-26T14:17:06,470 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/87e60260a333474c8a6b2114a325f019 2024-11-26T14:17:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/dc0308bdfc374b8e83b4d29bb70eaed4 is 50, key is test_row_0/C:col10/1732630624103/Put/seqid=0 2024-11-26T14:17:06,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742243_1419 (size=12001) 2024-11-26T14:17:06,497 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/dc0308bdfc374b8e83b4d29bb70eaed4 2024-11-26T14:17:06,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/c4577a01df914a82ac4c10d398dccdc5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5 2024-11-26T14:17:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630686488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,512 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T14:17:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/87e60260a333474c8a6b2114a325f019 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019 2024-11-26T14:17:06,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732630626484, max timestamp=1732630626509Num uniques 1095; 2024-11-26T14:17:06,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T14:17:06,517 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T14:17:06,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/dc0308bdfc374b8e83b4d29bb70eaed4 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4 2024-11-26T14:17:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732630626484, max timestamp=1732630626509Num uniques 1095; 2024-11-26T14:17:06,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T14:17:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732630626484, max timestamp=1732630626509Num uniques 1095; 2024-11-26T14:17:06,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T14:17:06,526 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4, entries=150, sequenceid=56, filesize=11.7 K 2024-11-26T14:17:06,529 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=53.67 KB/54960 for 4c88c058694b3dba456cf23dd49f28b4 in 924ms, sequenceid=56, compaction requested=true 2024-11-26T14:17:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T14:17:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:06,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=60.38 KB heapSize=153.09 KB 2024-11-26T14:17:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:06,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:06,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T14:17:06,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1740 sec 2024-11-26T14:17:06,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1870 sec 2024-11-26T14:17:06,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/047c624a939c4eae9a1056b213873efa is 50, key is test_row_0/A:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742244_1420 (size=19321) 2024-11-26T14:17:06,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/047c624a939c4eae9a1056b213873efa 2024-11-26T14:17:06,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/3bc91e3ed6ad4108aa502fab009f80b2 is 50, key is test_row_0/B:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:06,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742245_1421 (size=12151) 2024-11-26T14:17:06,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/3bc91e3ed6ad4108aa502fab009f80b2 2024-11-26T14:17:06,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/1cf1d6f95a864483902f3f0c084c4299 is 50, key is test_row_0/C:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:06,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742246_1422 (size=12151) 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630626590, max timestamp=1732630626613Num uniques 859; 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630626590, max timestamp=1732630626613Num uniques 859; 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630626590, max timestamp=1732630626613Num uniques 859; 2024-11-26T14:17:06,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:06,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630686649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630686661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630686662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630686663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630686664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630686774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630686774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630686783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630686783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630686784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630686983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630686983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/1cf1d6f95a864483902f3f0c084c4299 2024-11-26T14:17:07,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/047c624a939c4eae9a1056b213873efa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa 2024-11-26T14:17:07,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa, entries=300, sequenceid=148, filesize=18.9 K 2024-11-26T14:17:07,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/3bc91e3ed6ad4108aa502fab009f80b2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2 2024-11-26T14:17:07,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T14:17:07,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/1cf1d6f95a864483902f3f0c084c4299 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299 2024-11-26T14:17:07,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630687013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630687013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630687014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299, entries=150, sequenceid=148, filesize=11.9 K 2024-11-26T14:17:07,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~153.05 KB/156720, currentSize=127.47 KB/130530 for 4c88c058694b3dba456cf23dd49f28b4 in 535ms, sequenceid=148, compaction requested=true 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:07,065 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T14:17:07,068 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:07,073 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:07,073 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/A is initiating minor compaction (all files) 2024-11-26T14:17:07,073 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/A in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,073 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=58.6 K 2024-11-26T14:17:07,077 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87fd4f4905d941f1a61b6b0c8d038a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630623306 2024-11-26T14:17:07,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9396c9fc09c946bdb21e946a3faf341e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732630623419 2024-11-26T14:17:07,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4577a01df914a82ac4c10d398dccdc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732630624089 2024-11-26T14:17:07,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047c624a939c4eae9a1056b213873efa, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626484 2024-11-26T14:17:07,081 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:07,081 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/B is initiating minor compaction (all files) 2024-11-26T14:17:07,081 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/B in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,081 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=47.0 K 2024-11-26T14:17:07,081 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 399c00bf05234b14bfbefbfb7d8897cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630623306 2024-11-26T14:17:07,082 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1993208f844513abea060c7f8c508a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732630623419 2024-11-26T14:17:07,082 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e60260a333474c8a6b2114a325f019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732630624089 2024-11-26T14:17:07,083 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc91e3ed6ad4108aa502fab009f80b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626509 2024-11-26T14:17:07,107 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#A#compaction#353 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:07,108 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/524ff19656924ea0bd9546063fa716a8 is 50, key is test_row_0/A:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:07,111 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:07,113 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/c0b75a06060f419da826a13098e9ee73 is 50, key is test_row_0/B:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:07,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742247_1423 (size=12289) 2024-11-26T14:17:07,133 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/524ff19656924ea0bd9546063fa716a8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/524ff19656924ea0bd9546063fa716a8 2024-11-26T14:17:07,139 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/A of 4c88c058694b3dba456cf23dd49f28b4 into 524ff19656924ea0bd9546063fa716a8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:07,139 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:07,139 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/A, priority=12, startTime=1732630627065; duration=0sec 2024-11-26T14:17:07,139 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:07,139 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:A 2024-11-26T14:17:07,139 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:07,140 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:07,140 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/C is initiating minor compaction (all files) 2024-11-26T14:17:07,140 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/C in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,140 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=47.0 K 2024-11-26T14:17:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742248_1424 (size=12289) 2024-11-26T14:17:07,141 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2cd9089c67f40ccb9a7b78c8ef0b95a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732630623306 2024-11-26T14:17:07,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad364cbba224a7cbc5fa6b3e5aa1a09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732630623419 2024-11-26T14:17:07,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0308bdfc374b8e83b4d29bb70eaed4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732630624089 2024-11-26T14:17:07,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf1d6f95a864483902f3f0c084c4299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626509 2024-11-26T14:17:07,150 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/c0b75a06060f419da826a13098e9ee73 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c0b75a06060f419da826a13098e9ee73 2024-11-26T14:17:07,161 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#C#compaction#355 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:07,161 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/1c6c521c767142af8ef0761a4e2f3da1 is 50, key is test_row_0/C:col10/1732630626525/Put/seqid=0 2024-11-26T14:17:07,162 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/B of 4c88c058694b3dba456cf23dd49f28b4 into c0b75a06060f419da826a13098e9ee73(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:07,162 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:07,162 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/B, priority=12, startTime=1732630627065; duration=0sec 2024-11-26T14:17:07,162 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:07,162 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:B 2024-11-26T14:17:07,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742249_1425 (size=12289) 2024-11-26T14:17:07,194 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/1c6c521c767142af8ef0761a4e2f3da1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1c6c521c767142af8ef0761a4e2f3da1 2024-11-26T14:17:07,206 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/C of 4c88c058694b3dba456cf23dd49f28b4 into 1c6c521c767142af8ef0761a4e2f3da1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:07,206 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:07,206 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/C, priority=12, startTime=1732630627065; duration=0sec 2024-11-26T14:17:07,206 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:07,206 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:C 2024-11-26T14:17:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:07,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-26T14:17:07,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:07,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:07,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5212ec05203e41e3b34e4246c8386979 is 50, key is test_row_0/A:col10/1732630627322/Put/seqid=0 2024-11-26T14:17:07,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630687346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630687350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630687353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630687355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630687359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742250_1426 (size=16931) 2024-11-26T14:17:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630687463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630687474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630687474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630687474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630687474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T14:17:07,524 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:07,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-26T14:17:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:07,526 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:07,526 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:07,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630687674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630687693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630687693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630687693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630687693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:07,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5212ec05203e41e3b34e4246c8386979 2024-11-26T14:17:07,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/34d5784410884c3195cd51ee9db1381e is 50, key is test_row_0/B:col10/1732630627322/Put/seqid=0 2024-11-26T14:17:07,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742251_1427 (size=12151) 2024-11-26T14:17:07,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:07,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:07,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630687983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630688013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630688014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630688014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630688016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:08,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/34d5784410884c3195cd51ee9db1381e 2024-11-26T14:17:08,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/99624726744546bbb0ca0245b705e2f8 is 50, key is test_row_0/C:col10/1732630627322/Put/seqid=0 2024-11-26T14:17:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742252_1428 (size=12151) 2024-11-26T14:17:08,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630688504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630688533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630688533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630688533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630688533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:08,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:08,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/99624726744546bbb0ca0245b705e2f8 2024-11-26T14:17:08,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5212ec05203e41e3b34e4246c8386979 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979 2024-11-26T14:17:08,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979, entries=250, sequenceid=188, filesize=16.5 K 2024-11-26T14:17:08,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/34d5784410884c3195cd51ee9db1381e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e 2024-11-26T14:17:08,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-26T14:17:08,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/99624726744546bbb0ca0245b705e2f8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8 2024-11-26T14:17:08,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8, entries=150, sequenceid=188, filesize=11.9 K 2024-11-26T14:17:08,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4c88c058694b3dba456cf23dd49f28b4 in 1338ms, sequenceid=188, compaction requested=false 2024-11-26T14:17:08,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:08,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T14:17:08,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:08,755 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:08,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:08,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/280b566063854f9fb950a37776ef8006 is 50, key is test_row_0/A:col10/1732630627337/Put/seqid=0 2024-11-26T14:17:08,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742253_1429 (size=12151) 2024-11-26T14:17:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:11,279 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1171789504_22 at /127.0.0.1:50494 [Receiving block BP-17547768-172.17.0.3-1732630485263:blk_1073741833_1009] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 1754ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data1/, blockId=1073741833, seqno=2125 2024-11-26T14:17:11,280 INFO [AsyncFSWAL-0-hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc-prefix:19945b63c2b4,40025,1732630490995 {}] wal.AbstractFSWAL(1368): Slow sync cost: 1754 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK]] 2024-11-26T14:17:11,280 INFO [AsyncFSWAL-0-hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc-prefix:19945b63c2b4,40025,1732630490995 {}] wal.AbstractFSWAL(1368): Slow sync cost: 1719 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK]] 2024-11-26T14:17:11,280 INFO [AsyncFSWAL-0-hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc-prefix:19945b63c2b4,40025,1732630490995 {}] wal.AbstractFSWAL(1368): Slow sync cost: 1717 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46501,DS-0882fd91-32ba-4801-9f3d-ff3a821f0b34,DISK]] 2024-11-26T14:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:11,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. as already flushing 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630631313, max timestamp=1732630631319Num uniques 859; 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630631313, max timestamp=1732630631319Num uniques 859; 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732630631313, max timestamp=1732630631319Num uniques 859; 2024-11-26T14:17:11,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T14:17:11,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630691353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630691368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630691369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630691369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630691370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630691473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630691473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630691473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630691493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630691493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630691693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630691693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630691693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630691713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630691713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630692013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630692013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630692013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630692033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630692033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630692533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630692533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630692533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630692543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630692543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630693553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630693553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:13,565 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,565 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630693563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630693563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630693563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:13,570 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:13,570 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:13,571 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:15,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:15,584 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:15,584 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630695582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:15,595 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630695593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630695593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:15,605 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:15,605 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:17,550 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/280b566063854f9fb950a37776ef8006 2024-11-26T14:17:17,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/53586e1b23204d7188a52f8e61573885 is 50, key is test_row_0/B:col10/1732630627337/Put/seqid=0 2024-11-26T14:17:17,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742254_1430 (size=12151) 2024-11-26T14:17:17,973 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/53586e1b23204d7188a52f8e61573885 2024-11-26T14:17:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/a6e235c9b6504b3ab6607c3f9112d82b is 50, key is test_row_0/C:col10/1732630627337/Put/seqid=0 2024-11-26T14:17:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742255_1431 (size=12151) 2024-11-26T14:17:18,012 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/a6e235c9b6504b3ab6607c3f9112d82b 2024-11-26T14:17:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/280b566063854f9fb950a37776ef8006 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006 2024-11-26T14:17:18,042 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T14:17:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/53586e1b23204d7188a52f8e61573885 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885 2024-11-26T14:17:18,051 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T14:17:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/a6e235c9b6504b3ab6607c3f9112d82b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b 2024-11-26T14:17:18,058 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T14:17:18,060 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for 4c88c058694b3dba456cf23dd49f28b4 in 9306ms, sequenceid=197, compaction requested=true 2024-11-26T14:17:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:18,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-26T14:17:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-26T14:17:18,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T14:17:18,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 10.5350 sec 2024-11-26T14:17:18,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 10.5400 sec 2024-11-26T14:17:18,385 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:17:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:19,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:19,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:19,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/718f5c3a4c304d9088fb62b48003e227 is 50, key is test_row_0/A:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:19,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742256_1432 (size=14541) 2024-11-26T14:17:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630699627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630699628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630699631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732630699642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43634 deadline: 1732630699642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,656 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,656 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in 4c88c058694b3dba456cf23dd49f28b4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630699743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630699743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630699743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630699963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630699963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630699963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/718f5c3a4c304d9088fb62b48003e227 2024-11-26T14:17:20,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/258edaec1c25440ca70346d25c6619ae is 50, key is test_row_0/B:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:20,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742257_1433 (size=12151) 2024-11-26T14:17:20,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/258edaec1c25440ca70346d25c6619ae 2024-11-26T14:17:20,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/ed0340e680c34fb5b87957d9b78ed1e1 is 50, key is test_row_0/C:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:20,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742258_1434 (size=12151) 2024-11-26T14:17:20,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/ed0340e680c34fb5b87957d9b78ed1e1 2024-11-26T14:17:20,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/718f5c3a4c304d9088fb62b48003e227 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227 2024-11-26T14:17:20,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227, entries=200, sequenceid=235, filesize=14.2 K 2024-11-26T14:17:20,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/258edaec1c25440ca70346d25c6619ae as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae 2024-11-26T14:17:20,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T14:17:20,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/ed0340e680c34fb5b87957d9b78ed1e1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1 2024-11-26T14:17:20,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1, entries=150, sequenceid=235, filesize=11.9 K 2024-11-26T14:17:20,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 4c88c058694b3dba456cf23dd49f28b4 in 463ms, sequenceid=235, compaction requested=true 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:20,073 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:20,073 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:20,076 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:20,077 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/A is initiating minor compaction (all files) 2024-11-26T14:17:20,077 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/A in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:20,077 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/524ff19656924ea0bd9546063fa716a8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=54.6 K 2024-11-26T14:17:20,077 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:20,077 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/B is initiating minor compaction (all files) 2024-11-26T14:17:20,077 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/B in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:20,077 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c0b75a06060f419da826a13098e9ee73, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=47.6 K 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524ff19656924ea0bd9546063fa716a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626509 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b75a06060f419da826a13098e9ee73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626509 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5212ec05203e41e3b34e4246c8386979, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630626660 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d5784410884c3195cd51ee9db1381e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630626660 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280b566063854f9fb950a37776ef8006, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630627337 2024-11-26T14:17:20,078 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 53586e1b23204d7188a52f8e61573885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630627337 2024-11-26T14:17:20,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 718f5c3a4c304d9088fb62b48003e227, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:20,079 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 258edaec1c25440ca70346d25c6619ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:20,091 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#B#compaction#365 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:20,091 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/3f876cd003f34afa8635a6394284ef7d is 50, key is test_row_0/B:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:20,095 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:20,099 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/47db97e3f162459a94b50fbe15214706 is 50, key is test_row_0/A:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:20,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742259_1435 (size=12425) 2024-11-26T14:17:20,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742260_1436 (size=12425) 2024-11-26T14:17:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:20,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:20,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:20,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/4e8a29772a9c43c5b4584949519c0a2f is 50, key is test_row_0/A:col10/1732630640279/Put/seqid=0 2024-11-26T14:17:20,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742261_1437 (size=16931) 2024-11-26T14:17:20,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/4e8a29772a9c43c5b4584949519c0a2f 2024-11-26T14:17:20,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/7c386231d6cd4003be1be7674225bc85 is 50, key is test_row_0/B:col10/1732630640279/Put/seqid=0 2024-11-26T14:17:20,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742262_1438 (size=12151) 2024-11-26T14:17:20,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/7c386231d6cd4003be1be7674225bc85 2024-11-26T14:17:20,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/772590d0031543f78edc8b17aab5dbbc is 50, key is test_row_0/C:col10/1732630640279/Put/seqid=0 2024-11-26T14:17:20,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742263_1439 (size=12151) 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630640364, max timestamp=1732630640382Num uniques 859; 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630640364, max timestamp=1732630640382Num uniques 859; 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630640364, max timestamp=1732630640382Num uniques 859; 2024-11-26T14:17:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:20,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630700452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630700453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630700463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,507 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/3f876cd003f34afa8635a6394284ef7d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3f876cd003f34afa8635a6394284ef7d 2024-11-26T14:17:20,511 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/B of 4c88c058694b3dba456cf23dd49f28b4 into 3f876cd003f34afa8635a6394284ef7d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:20,511 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:20,511 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/B, priority=12, startTime=1732630640073; duration=0sec 2024-11-26T14:17:20,511 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:20,512 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:B 2024-11-26T14:17:20,512 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:20,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:20,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/C is initiating minor compaction (all files) 2024-11-26T14:17:20,513 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/C in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:20,513 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1c6c521c767142af8ef0761a4e2f3da1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=47.6 K 2024-11-26T14:17:20,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6c521c767142af8ef0761a4e2f3da1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732630626509 2024-11-26T14:17:20,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 99624726744546bbb0ca0245b705e2f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630626660 2024-11-26T14:17:20,514 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a6e235c9b6504b3ab6607c3f9112d82b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732630627337 2024-11-26T14:17:20,514 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0340e680c34fb5b87957d9b78ed1e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:20,517 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/47db97e3f162459a94b50fbe15214706 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/47db97e3f162459a94b50fbe15214706 2024-11-26T14:17:20,521 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/A of 4c88c058694b3dba456cf23dd49f28b4 into 47db97e3f162459a94b50fbe15214706(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:20,521 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:20,521 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/A, priority=12, startTime=1732630640073; duration=0sec 2024-11-26T14:17:20,521 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:20,521 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:A 2024-11-26T14:17:20,523 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:20,523 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/a843fda3a0184cd8a89b64a8bbbfa123 is 50, key is test_row_0/C:col10/1732630631352/Put/seqid=0 2024-11-26T14:17:20,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742264_1440 (size=12425) 2024-11-26T14:17:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630700573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630700583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630700583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/772590d0031543f78edc8b17aab5dbbc 2024-11-26T14:17:20,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/4e8a29772a9c43c5b4584949519c0a2f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f 2024-11-26T14:17:20,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f, entries=250, sequenceid=246, filesize=16.5 K 2024-11-26T14:17:20,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/7c386231d6cd4003be1be7674225bc85 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85 2024-11-26T14:17:20,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T14:17:20,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/772590d0031543f78edc8b17aab5dbbc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc 2024-11-26T14:17:20,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T14:17:20,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4c88c058694b3dba456cf23dd49f28b4 in 452ms, sequenceid=246, compaction requested=false 2024-11-26T14:17:20,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:20,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:20,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:20,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/003cf41ec0264cd99073deb19f5da21f is 50, key is test_row_0/A:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:20,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742265_1441 (size=12301) 2024-11-26T14:17:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630700872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630700872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630700872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:20,938 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/a843fda3a0184cd8a89b64a8bbbfa123 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a843fda3a0184cd8a89b64a8bbbfa123 2024-11-26T14:17:20,942 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/C of 4c88c058694b3dba456cf23dd49f28b4 into a843fda3a0184cd8a89b64a8bbbfa123(size=12.1 K), total size for store is 24 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:20,942 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:20,942 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/C, priority=12, startTime=1732630640073; duration=0sec 2024-11-26T14:17:20,942 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:20,942 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:C 2024-11-26T14:17:20,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630700983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630700993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630700993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630701202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630701202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630701203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/003cf41ec0264cd99073deb19f5da21f 2024-11-26T14:17:21,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 is 50, key is test_row_0/B:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:21,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742266_1442 (size=12301) 2024-11-26T14:17:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630701523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630701523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630701523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:21,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 2024-11-26T14:17:21,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/963a423fc3ea4f91b8834b7a148e0137 is 50, key is test_row_0/C:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:21,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742267_1443 (size=12301) 2024-11-26T14:17:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630702043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630702043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630702043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:22,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/963a423fc3ea4f91b8834b7a148e0137 2024-11-26T14:17:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/003cf41ec0264cd99073deb19f5da21f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f 2024-11-26T14:17:22,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T14:17:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 2024-11-26T14:17:22,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T14:17:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/963a423fc3ea4f91b8834b7a148e0137 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137 2024-11-26T14:17:22,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137, entries=150, sequenceid=285, filesize=12.0 K 2024-11-26T14:17:22,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for 4c88c058694b3dba456cf23dd49f28b4 in 1280ms, sequenceid=285, compaction requested=true 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:22,078 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:22,078 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:22,079 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:22,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:22,079 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/B is initiating minor compaction (all files) 2024-11-26T14:17:22,079 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/A is initiating minor compaction (all files) 2024-11-26T14:17:22,079 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/B in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:22,079 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/A in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:22,079 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3f876cd003f34afa8635a6394284ef7d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=36.0 K 2024-11-26T14:17:22,079 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/47db97e3f162459a94b50fbe15214706, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=40.7 K 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f876cd003f34afa8635a6394284ef7d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47db97e3f162459a94b50fbe15214706, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8a29772a9c43c5b4584949519c0a2f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732630639627 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c386231d6cd4003be1be7674225bc85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732630639627 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec3cceb4bf54f93a1fbd9b25bd90b24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:22,080 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003cf41ec0264cd99073deb19f5da21f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:22,087 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#A#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:22,087 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/3ce41402993d49b798331f671f8b9014 is 50, key is test_row_0/A:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:22,090 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:22,090 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/b2a00545a01e484181170e211f98156e is 50, key is test_row_0/B:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742268_1444 (size=12677) 2024-11-26T14:17:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742269_1445 (size=12677) 2024-11-26T14:17:22,107 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/b2a00545a01e484181170e211f98156e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/b2a00545a01e484181170e211f98156e 2024-11-26T14:17:22,111 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/B of 4c88c058694b3dba456cf23dd49f28b4 into b2a00545a01e484181170e211f98156e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:22,111 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:22,111 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/B, priority=13, startTime=1732630642078; duration=0sec 2024-11-26T14:17:22,111 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:22,111 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:B 2024-11-26T14:17:22,111 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:22,113 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:22,113 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/C is initiating minor compaction (all files) 2024-11-26T14:17:22,113 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/C in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:22,113 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a843fda3a0184cd8a89b64a8bbbfa123, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=36.0 K 2024-11-26T14:17:22,114 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a843fda3a0184cd8a89b64a8bbbfa123, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732630631352 2024-11-26T14:17:22,114 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 772590d0031543f78edc8b17aab5dbbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732630639627 2024-11-26T14:17:22,114 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 963a423fc3ea4f91b8834b7a148e0137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:22,120 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:22,120 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/c7ecbbc45e8a4a5898a8f2423a99257d is 50, key is test_row_0/C:col10/1732630640450/Put/seqid=0 2024-11-26T14:17:22,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742270_1446 (size=12677) 2024-11-26T14:17:22,501 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/3ce41402993d49b798331f671f8b9014 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/3ce41402993d49b798331f671f8b9014 2024-11-26T14:17:22,505 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/A of 4c88c058694b3dba456cf23dd49f28b4 into 3ce41402993d49b798331f671f8b9014(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:22,505 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:22,505 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/A, priority=13, startTime=1732630642078; duration=0sec 2024-11-26T14:17:22,505 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:22,505 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:A 2024-11-26T14:17:22,529 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/c7ecbbc45e8a4a5898a8f2423a99257d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/c7ecbbc45e8a4a5898a8f2423a99257d 2024-11-26T14:17:22,533 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/C of 4c88c058694b3dba456cf23dd49f28b4 into c7ecbbc45e8a4a5898a8f2423a99257d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:22,533 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:22,533 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/C, priority=13, startTime=1732630642078; duration=0sec 2024-11-26T14:17:22,533 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:22,533 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:C 2024-11-26T14:17:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:23,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:23,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/a2fa8547b9ed474f933e80664d207c23 is 50, key is test_row_0/A:col10/1732630640870/Put/seqid=0 2024-11-26T14:17:23,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742271_1447 (size=14741) 2024-11-26T14:17:23,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/a2fa8547b9ed474f933e80664d207c23 2024-11-26T14:17:23,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/91438c48ed314cdcafb079ca1ed7b479 is 50, key is test_row_0/B:col10/1732630640870/Put/seqid=0 2024-11-26T14:17:23,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742272_1448 (size=12301) 2024-11-26T14:17:23,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/91438c48ed314cdcafb079ca1ed7b479 2024-11-26T14:17:23,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/18789d364e054733b0203ee8d309488f is 50, key is test_row_0/C:col10/1732630640870/Put/seqid=0 2024-11-26T14:17:23,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742273_1449 (size=12301) 2024-11-26T14:17:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630703111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630703111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630703112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630703223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630703223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630703233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,363 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-26T14:17:23,363 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,363 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,363 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,366 INFO [Thread-1879 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1879. 2024-11-26T14:17:23,366 DEBUG [Thread-1879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,366 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,366 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,366 INFO [Thread-1881 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1881. 2024-11-26T14:17:23,367 DEBUG [Thread-1881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,367 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,367 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,373 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-26T14:17:23,374 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,374 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,374 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,375 INFO [Thread-1877 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1877. 2024-11-26T14:17:23,376 DEBUG [Thread-1877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,376 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,376 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43602 deadline: 1732630703434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43674 deadline: 1732630703434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43620 deadline: 1732630703444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 , the old value is region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., hostname=19945b63c2b4,40025,1732630490995, seqNum=2 because the exception is null or not the one we care about 2024-11-26T14:17:23,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/18789d364e054733b0203ee8d309488f 2024-11-26T14:17:23,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/a2fa8547b9ed474f933e80664d207c23 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23 2024-11-26T14:17:23,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23, entries=200, sequenceid=302, filesize=14.4 K 2024-11-26T14:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/91438c48ed314cdcafb079ca1ed7b479 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479 2024-11-26T14:17:23,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479, entries=150, sequenceid=302, filesize=12.0 K 2024-11-26T14:17:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/18789d364e054733b0203ee8d309488f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f 2024-11-26T14:17:23,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f, entries=150, sequenceid=302, filesize=12.0 K 2024-11-26T14:17:23,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4c88c058694b3dba456cf23dd49f28b4 in 461ms, sequenceid=302, compaction requested=false 2024-11-26T14:17:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:23,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T14:17:23,748 INFO [Thread-1866 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1866. 2024-11-26T14:17:23,748 INFO [Thread-1868 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1868. 2024-11-26T14:17:23,748 DEBUG [Thread-1866 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,749 DEBUG [Thread-1866 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,749 DEBUG [Thread-1868 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:23,749 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,749 DEBUG [Thread-1866 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,749 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:23,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,754 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-26T14:17:23,754 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:23,754 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,754 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:23,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:23,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5b007f133717461fb913557b5bff0b32 is 50, key is test_row_0/A:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:23,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742274_1450 (size=12301) 2024-11-26T14:17:24,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5b007f133717461fb913557b5bff0b32 2024-11-26T14:17:24,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/c235b050978948ff959945f8d4774c0d is 50, key is test_row_0/B:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742275_1451 (size=12301) 2024-11-26T14:17:24,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/c235b050978948ff959945f8d4774c0d 2024-11-26T14:17:24,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/3f5f52881cb347fdb6b932caad31f631 is 50, key is test_row_0/C:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:24,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742276_1452 (size=12301) 2024-11-26T14:17:24,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/3f5f52881cb347fdb6b932caad31f631 2024-11-26T14:17:25,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/5b007f133717461fb913557b5bff0b32 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32 2024-11-26T14:17:25,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32, entries=150, sequenceid=323, filesize=12.0 K 2024-11-26T14:17:25,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/c235b050978948ff959945f8d4774c0d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d 2024-11-26T14:17:25,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d, entries=150, sequenceid=323, filesize=12.0 K 2024-11-26T14:17:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/3f5f52881cb347fdb6b932caad31f631 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631 2024-11-26T14:17:25,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631, entries=150, sequenceid=323, filesize=12.0 K 2024-11-26T14:17:25,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for 4c88c058694b3dba456cf23dd49f28b4 in 1275ms, sequenceid=323, compaction requested=true 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:25,023 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:25,023 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:25,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c88c058694b3dba456cf23dd49f28b4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:25,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:25,024 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:25,024 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:25,024 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/B is initiating minor compaction (all files) 2024-11-26T14:17:25,024 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/A is initiating minor compaction (all files) 2024-11-26T14:17:25,024 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/B in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:25,024 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/A in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:25,025 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/b2a00545a01e484181170e211f98156e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=36.4 K 2024-11-26T14:17:25,025 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/3ce41402993d49b798331f671f8b9014, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=38.8 K 2024-11-26T14:17:25,025 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a00545a01e484181170e211f98156e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:25,025 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce41402993d49b798331f671f8b9014, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:25,025 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 91438c48ed314cdcafb079ca1ed7b479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732630640870 2024-11-26T14:17:25,025 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2fa8547b9ed474f933e80664d207c23, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732630640851 2024-11-26T14:17:25,026 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b007f133717461fb913557b5bff0b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732630643111 2024-11-26T14:17:25,026 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c235b050978948ff959945f8d4774c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732630643111 2024-11-26T14:17:25,034 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:25,034 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#A#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:25,035 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/014bb40f26ef481b9ea70260b32d96f7 is 50, key is test_row_0/A:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:25,035 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/104bdb99e7ff4369acca3b6ffbee6ac9 is 50, key is test_row_0/B:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:25,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742277_1453 (size=12779) 2024-11-26T14:17:25,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742278_1454 (size=12779) 2024-11-26T14:17:25,448 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/104bdb99e7ff4369acca3b6ffbee6ac9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/104bdb99e7ff4369acca3b6ffbee6ac9 2024-11-26T14:17:25,448 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/014bb40f26ef481b9ea70260b32d96f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/014bb40f26ef481b9ea70260b32d96f7 2024-11-26T14:17:25,453 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/B of 4c88c058694b3dba456cf23dd49f28b4 into 104bdb99e7ff4369acca3b6ffbee6ac9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:25,453 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/A of 4c88c058694b3dba456cf23dd49f28b4 into 014bb40f26ef481b9ea70260b32d96f7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:25,453 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/B, priority=13, startTime=1732630645023; duration=0sec 2024-11-26T14:17:25,453 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/A, priority=13, startTime=1732630645023; duration=0sec 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:B 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:A 2024-11-26T14:17:25,453 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:25,454 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:25,454 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): 4c88c058694b3dba456cf23dd49f28b4/C is initiating minor compaction (all files) 2024-11-26T14:17:25,454 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c88c058694b3dba456cf23dd49f28b4/C in TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:25,454 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/c7ecbbc45e8a4a5898a8f2423a99257d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp, totalSize=36.4 K 2024-11-26T14:17:25,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ecbbc45e8a4a5898a8f2423a99257d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732630640450 2024-11-26T14:17:25,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 18789d364e054733b0203ee8d309488f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732630640870 2024-11-26T14:17:25,455 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5f52881cb347fdb6b932caad31f631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732630643111 2024-11-26T14:17:25,461 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c88c058694b3dba456cf23dd49f28b4#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:25,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/ca3ded9a160c4fd1a13580b5320a87b9 is 50, key is test_row_0/C:col10/1732630643744/Put/seqid=0 2024-11-26T14:17:25,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742279_1455 (size=12779) 2024-11-26T14:17:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T14:17:25,764 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:27,110 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/ca3ded9a160c4fd1a13580b5320a87b9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ca3ded9a160c4fd1a13580b5320a87b9 2024-11-26T14:17:27,115 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c88c058694b3dba456cf23dd49f28b4/C of 4c88c058694b3dba456cf23dd49f28b4 into ca3ded9a160c4fd1a13580b5320a87b9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-26T14:17:27,115 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c88c058694b3dba456cf23dd49f28b4: 2024-11-26T14:17:27,115 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., storeName=4c88c058694b3dba456cf23dd49f28b4/C, priority=13, startTime=1732630645023; duration=1sec 2024-11-26T14:17:27,116 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:27,116 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c88c058694b3dba456cf23dd49f28b4:C 2024-11-26T14:17:29,706 INFO [Thread-1864 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1864. 2024-11-26T14:17:29,706 INFO [Thread-1862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1862. 2024-11-26T14:17:29,706 DEBUG [Thread-1862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:29,706 DEBUG [Thread-1864 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:29,706 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,706 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,706 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,706 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:29,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:29,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:17:29,706 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 824 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2472 rows 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 820 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2460 rows 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 835 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2496 rows 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 834 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2502 rows 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 829 2024-11-26T14:17:29,707 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2487 rows 2024-11-26T14:17:29,707 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:17:29,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:17:29,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:29,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:29,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:17:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:29,713 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630649712"}]},"ts":"1732630649712"} 2024-11-26T14:17:29,714 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:17:29,714 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:17:29,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:17:29,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, UNASSIGN}] 2024-11-26T14:17:29,718 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, UNASSIGN 2024-11-26T14:17:29,718 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4c88c058694b3dba456cf23dd49f28b4, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:29,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, UNASSIGN because future has completed 2024-11-26T14:17:29,720 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:17:29,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:17:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:29,873 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 4c88c058694b3dba456cf23dd49f28b4, disabling compactions & flushes 2024-11-26T14:17:29,873 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. after waiting 0 ms 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:29,873 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 4c88c058694b3dba456cf23dd49f28b4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=A 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=B 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c88c058694b3dba456cf23dd49f28b4, store=C 2024-11-26T14:17:29,873 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:29,877 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/8947e00898324dfaacad80d4d5687396 is 50, key is test_row_0/A:col10/1732630649704/Put/seqid=0 2024-11-26T14:17:29,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742280_1456 (size=9857) 2024-11-26T14:17:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:30,281 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/8947e00898324dfaacad80d4d5687396 2024-11-26T14:17:30,288 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/166c0e5090a6487dad3b30cb4f2b38fc is 50, key is test_row_0/B:col10/1732630649704/Put/seqid=0 2024-11-26T14:17:30,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742281_1457 (size=9857) 2024-11-26T14:17:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:30,692 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/166c0e5090a6487dad3b30cb4f2b38fc 2024-11-26T14:17:30,699 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/467f254e8d974976b30a0b338089a497 is 50, key is test_row_0/C:col10/1732630649704/Put/seqid=0 2024-11-26T14:17:30,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742282_1458 (size=9857) 2024-11-26T14:17:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:31,104 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/467f254e8d974976b30a0b338089a497 2024-11-26T14:17:31,111 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/A/8947e00898324dfaacad80d4d5687396 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/8947e00898324dfaacad80d4d5687396 2024-11-26T14:17:31,116 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/8947e00898324dfaacad80d4d5687396, entries=100, sequenceid=332, filesize=9.6 K 2024-11-26T14:17:31,117 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/B/166c0e5090a6487dad3b30cb4f2b38fc as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/166c0e5090a6487dad3b30cb4f2b38fc 2024-11-26T14:17:31,122 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/166c0e5090a6487dad3b30cb4f2b38fc, entries=100, sequenceid=332, filesize=9.6 K 2024-11-26T14:17:31,122 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/.tmp/C/467f254e8d974976b30a0b338089a497 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/467f254e8d974976b30a0b338089a497 2024-11-26T14:17:31,126 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/467f254e8d974976b30a0b338089a497, entries=100, sequenceid=332, filesize=9.6 K 2024-11-26T14:17:31,127 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4c88c058694b3dba456cf23dd49f28b4 in 1254ms, sequenceid=332, compaction requested=false 2024-11-26T14:17:31,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/524ff19656924ea0bd9546063fa716a8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/47db97e3f162459a94b50fbe15214706, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/3ce41402993d49b798331f671f8b9014, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32] to archive 2024-11-26T14:17:31,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:17:31,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/87fd4f4905d941f1a61b6b0c8d038a4a 2024-11-26T14:17:31,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/9396c9fc09c946bdb21e946a3faf341e 2024-11-26T14:17:31,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/c4577a01df914a82ac4c10d398dccdc5 2024-11-26T14:17:31,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/047c624a939c4eae9a1056b213873efa 2024-11-26T14:17:31,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/524ff19656924ea0bd9546063fa716a8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/524ff19656924ea0bd9546063fa716a8 2024-11-26T14:17:31,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5212ec05203e41e3b34e4246c8386979 2024-11-26T14:17:31,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/280b566063854f9fb950a37776ef8006 2024-11-26T14:17:31,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/718f5c3a4c304d9088fb62b48003e227 2024-11-26T14:17:31,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/47db97e3f162459a94b50fbe15214706 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/47db97e3f162459a94b50fbe15214706 2024-11-26T14:17:31,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/4e8a29772a9c43c5b4584949519c0a2f 2024-11-26T14:17:31,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/3ce41402993d49b798331f671f8b9014 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/3ce41402993d49b798331f671f8b9014 2024-11-26T14:17:31,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/003cf41ec0264cd99073deb19f5da21f 2024-11-26T14:17:31,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/a2fa8547b9ed474f933e80664d207c23 2024-11-26T14:17:31,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/5b007f133717461fb913557b5bff0b32 2024-11-26T14:17:31,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c0b75a06060f419da826a13098e9ee73, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3f876cd003f34afa8635a6394284ef7d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/b2a00545a01e484181170e211f98156e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d] to archive 2024-11-26T14:17:31,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:17:31,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/399c00bf05234b14bfbefbfb7d8897cf 2024-11-26T14:17:31,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/0e1993208f844513abea060c7f8c508a 2024-11-26T14:17:31,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/87e60260a333474c8a6b2114a325f019 2024-11-26T14:17:31,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c0b75a06060f419da826a13098e9ee73 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c0b75a06060f419da826a13098e9ee73 2024-11-26T14:17:31,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3bc91e3ed6ad4108aa502fab009f80b2 2024-11-26T14:17:31,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/34d5784410884c3195cd51ee9db1381e 2024-11-26T14:17:31,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/53586e1b23204d7188a52f8e61573885 2024-11-26T14:17:31,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3f876cd003f34afa8635a6394284ef7d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/3f876cd003f34afa8635a6394284ef7d 2024-11-26T14:17:31,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/258edaec1c25440ca70346d25c6619ae 2024-11-26T14:17:31,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/7c386231d6cd4003be1be7674225bc85 2024-11-26T14:17:31,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/b2a00545a01e484181170e211f98156e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/b2a00545a01e484181170e211f98156e 2024-11-26T14:17:31,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/8ec3cceb4bf54f93a1fbd9b25bd90b24 2024-11-26T14:17:31,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/91438c48ed314cdcafb079ca1ed7b479 2024-11-26T14:17:31,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/c235b050978948ff959945f8d4774c0d 2024-11-26T14:17:31,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1c6c521c767142af8ef0761a4e2f3da1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a843fda3a0184cd8a89b64a8bbbfa123, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/c7ecbbc45e8a4a5898a8f2423a99257d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631] to archive 2024-11-26T14:17:31,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:17:31,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/d2cd9089c67f40ccb9a7b78c8ef0b95a 2024-11-26T14:17:31,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/0ad364cbba224a7cbc5fa6b3e5aa1a09 2024-11-26T14:17:31,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/dc0308bdfc374b8e83b4d29bb70eaed4 2024-11-26T14:17:31,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1c6c521c767142af8ef0761a4e2f3da1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1c6c521c767142af8ef0761a4e2f3da1 2024-11-26T14:17:31,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/1cf1d6f95a864483902f3f0c084c4299 2024-11-26T14:17:31,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/99624726744546bbb0ca0245b705e2f8 2024-11-26T14:17:31,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a6e235c9b6504b3ab6607c3f9112d82b 2024-11-26T14:17:31,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a843fda3a0184cd8a89b64a8bbbfa123 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/a843fda3a0184cd8a89b64a8bbbfa123 2024-11-26T14:17:31,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ed0340e680c34fb5b87957d9b78ed1e1 2024-11-26T14:17:31,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/772590d0031543f78edc8b17aab5dbbc 2024-11-26T14:17:31,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/c7ecbbc45e8a4a5898a8f2423a99257d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/c7ecbbc45e8a4a5898a8f2423a99257d 2024-11-26T14:17:31,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/963a423fc3ea4f91b8834b7a148e0137 2024-11-26T14:17:31,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/18789d364e054733b0203ee8d309488f 2024-11-26T14:17:31,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/3f5f52881cb347fdb6b932caad31f631 2024-11-26T14:17:31,256 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=1 2024-11-26T14:17:31,258 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. 2024-11-26T14:17:31,259 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 4c88c058694b3dba456cf23dd49f28b4: Waiting for close lock at 1732630649873Running coprocessor pre-close hooks at 1732630649873Disabling compacts and flushes for region at 1732630649873Disabling writes for close at 1732630649873Obtaining lock to block concurrent updates at 1732630649873Preparing flush snapshotting stores in 4c88c058694b3dba456cf23dd49f28b4 at 1732630649873Finished memstore snapshotting TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732630649874 (+1 ms)Flushing stores of TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4. at 1732630649874Flushing 4c88c058694b3dba456cf23dd49f28b4/A: creating writer at 1732630649874Flushing 4c88c058694b3dba456cf23dd49f28b4/A: appending metadata at 1732630649876 (+2 ms)Flushing 4c88c058694b3dba456cf23dd49f28b4/A: closing flushed file at 1732630649876Flushing 4c88c058694b3dba456cf23dd49f28b4/B: creating writer at 1732630650286 (+410 ms)Flushing 4c88c058694b3dba456cf23dd49f28b4/B: appending metadata at 1732630650288 (+2 ms)Flushing 4c88c058694b3dba456cf23dd49f28b4/B: closing flushed file at 1732630650288Flushing 4c88c058694b3dba456cf23dd49f28b4/C: creating writer at 1732630650696 (+408 ms)Flushing 4c88c058694b3dba456cf23dd49f28b4/C: appending metadata at 1732630650698 (+2 ms)Flushing 4c88c058694b3dba456cf23dd49f28b4/C: closing flushed file at 1732630650698Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b10e1b6: reopening flushed file at 1732630651108 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a9b20e0: reopening flushed file at 1732630651116 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f045e1d: reopening flushed file at 1732630651122 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4c88c058694b3dba456cf23dd49f28b4 in 1254ms, sequenceid=332, compaction requested=false at 1732630651127 (+5 ms)Writing region close event to WAL at 1732630651243 (+116 ms)Running coprocessor post-close hooks at 1732630651258 (+15 ms)Closed at 1732630651258 2024-11-26T14:17:31,261 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:31,261 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4c88c058694b3dba456cf23dd49f28b4, regionState=CLOSED 2024-11-26T14:17:31,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:17:31,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T14:17:31,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c88c058694b3dba456cf23dd49f28b4, server=19945b63c2b4,40025,1732630490995 in 1.5510 sec 2024-11-26T14:17:31,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-26T14:17:31,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c88c058694b3dba456cf23dd49f28b4, UNASSIGN in 1.5560 sec 2024-11-26T14:17:31,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T14:17:31,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5630 sec 2024-11-26T14:17:31,283 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630651282"}]},"ts":"1732630651282"} 2024-11-26T14:17:31,285 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:17:31,285 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:17:31,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5770 sec 2024-11-26T14:17:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T14:17:31,864 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:31,864 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:17:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,866 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,866 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T14:17:31,869 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:31,870 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/recovered.edits] 2024-11-26T14:17:31,872 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/014bb40f26ef481b9ea70260b32d96f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/014bb40f26ef481b9ea70260b32d96f7 2024-11-26T14:17:31,873 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/8947e00898324dfaacad80d4d5687396 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/A/8947e00898324dfaacad80d4d5687396 2024-11-26T14:17:31,875 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/104bdb99e7ff4369acca3b6ffbee6ac9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/104bdb99e7ff4369acca3b6ffbee6ac9 2024-11-26T14:17:31,876 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/166c0e5090a6487dad3b30cb4f2b38fc to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/B/166c0e5090a6487dad3b30cb4f2b38fc 2024-11-26T14:17:31,877 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/467f254e8d974976b30a0b338089a497 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/467f254e8d974976b30a0b338089a497 2024-11-26T14:17:31,878 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ca3ded9a160c4fd1a13580b5320a87b9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/C/ca3ded9a160c4fd1a13580b5320a87b9 2024-11-26T14:17:31,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/recovered.edits/335.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4/recovered.edits/335.seqid 2024-11-26T14:17:31,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/4c88c058694b3dba456cf23dd49f28b4 2024-11-26T14:17:31,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:17:31,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,884 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:17:31,885 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:17:31,886 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,886 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:17:31,886 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630651886"}]},"ts":"9223372036854775807"} 2024-11-26T14:17:31,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:17:31,888 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c88c058694b3dba456cf23dd49f28b4, NAME => 'TestAcidGuarantees,,1732630604975.4c88c058694b3dba456cf23dd49f28b4.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:17:31,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:17:31,888 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630651888"}]},"ts":"9223372036854775807"} 2024-11-26T14:17:31,890 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:17:31,890 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-26T14:17:31,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T14:17:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:17:31,975 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:31,983 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 252), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1105 (was 1156), ProcessCount=11 (was 11), AvailableMemoryMB=6390 (was 5167) - AvailableMemoryMB LEAK? - 2024-11-26T14:17:31,992 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1105, ProcessCount=11, AvailableMemoryMB=6388 2024-11-26T14:17:31,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:17:31,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:17:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:31,997 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:17:31,997 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:31,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-26T14:17:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:31,998 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:17:32,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742283_1459 (size=1061) 2024-11-26T14:17:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:32,411 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc 2024-11-26T14:17:32,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742284_1460 (size=53) 2024-11-26T14:17:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:32,821 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:17:32,821 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a053eb1b7cd48562bf65d3c9969f8a5b, disabling compactions & flushes 2024-11-26T14:17:32,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:32,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:32,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. after waiting 0 ms 2024-11-26T14:17:32,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:32,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:32,822 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a053eb1b7cd48562bf65d3c9969f8a5b: Waiting for close lock at 1732630652821Disabling compacts and flushes for region at 1732630652821Disabling writes for close at 1732630652822 (+1 ms)Writing region close event to WAL at 1732630652822Closed at 1732630652822 2024-11-26T14:17:32,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:17:32,824 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732630652823"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732630652823"}]},"ts":"1732630652823"} 2024-11-26T14:17:32,829 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:17:32,831 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:17:32,831 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630652831"}]},"ts":"1732630652831"} 2024-11-26T14:17:32,835 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T14:17:32,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, ASSIGN}] 2024-11-26T14:17:32,838 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, ASSIGN 2024-11-26T14:17:32,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, ASSIGN; state=OFFLINE, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=false 2024-11-26T14:17:32,990 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:32,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, ASSIGN because future has completed 2024-11-26T14:17:32,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:17:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:33,169 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:33,169 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:17:33,169 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,170 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:17:33,170 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,170 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,188 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,216 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:33,217 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName A 2024-11-26T14:17:33,217 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:33,224 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:33,224 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,236 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:33,237 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName B 2024-11-26T14:17:33,237 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:33,240 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:33,241 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,247 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:33,247 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName C 2024-11-26T14:17:33,247 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:33,252 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:33,253 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:33,253 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,260 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,268 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,284 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,284 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,291 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:17:33,299 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,300 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:17:33,301 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened a053eb1b7cd48562bf65d3c9969f8a5b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59787173, jitterRate=-0.10910169780254364}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:17:33,301 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:33,301 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for a053eb1b7cd48562bf65d3c9969f8a5b: Running coprocessor pre-open hook at 1732630653170Writing region info on filesystem at 1732630653170Initializing all the Stores at 1732630653182 (+12 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630653182Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630653187 (+5 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630653187Cleaning up temporary data from old regions at 1732630653284 (+97 ms)Running coprocessor post-open hooks at 1732630653301 (+17 ms)Region opened successfully at 1732630653301 2024-11-26T14:17:33,302 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., pid=112, masterSystemTime=1732630653148 2024-11-26T14:17:33,303 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:33,303 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:33,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:33,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:17:33,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T14:17:33,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 in 322 msec 2024-11-26T14:17:33,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-26T14:17:33,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, ASSIGN in 480 msec 2024-11-26T14:17:33,319 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:17:33,319 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630653319"}]},"ts":"1732630653319"} 2024-11-26T14:17:33,320 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T14:17:33,329 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:17:33,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3350 sec 2024-11-26T14:17:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-26T14:17:34,144 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:34,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:34,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:34,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:34,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:34,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@961586e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:34,155 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222dd78e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:34,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:34,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:34,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:34,165 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:17:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,37705,1732630489590 2024-11-26T14:17:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@692d5c5f 2024-11-26T14:17:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:17:34,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:17:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T14:17:34,170 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:17:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T14:17:34,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742285_1461 (size=1097) 2024-11-26T14:17:34,597 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T14:17:34,597 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T14:17:34,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:17:34,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE}] 2024-11-26T14:17:34,617 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE 2024-11-26T14:17:34,617 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:34,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE because future has completed 2024-11-26T14:17:34,620 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:17:34,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:17:34,773 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:34,773 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:17:34,773 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing a053eb1b7cd48562bf65d3c9969f8a5b, disabling compactions & flushes 2024-11-26T14:17:34,773 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:34,773 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:34,773 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. after waiting 0 ms 2024-11-26T14:17:34,773 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:34,775 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T14:17:34,776 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:34,776 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for a053eb1b7cd48562bf65d3c9969f8a5b: Waiting for close lock at 1732630654773Running coprocessor pre-close hooks at 1732630654773Disabling compacts and flushes for region at 1732630654773Disabling writes for close at 1732630654773Writing region close event to WAL at 1732630654773Running coprocessor post-close hooks at 1732630654776 (+3 ms)Closed at 1732630654776 2024-11-26T14:17:34,777 WARN [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionServer(3299): Not adding moved region record: a053eb1b7cd48562bf65d3c9969f8a5b to self. 2024-11-26T14:17:34,778 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:34,778 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=CLOSED 2024-11-26T14:17:34,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:17:34,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T14:17:34,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 in 164 msec 2024-11-26T14:17:34,787 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE; state=CLOSED, location=19945b63c2b4,40025,1732630490995; forceNewPlan=false, retain=true 2024-11-26T14:17:34,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=OPENING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:34,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE because future has completed 2024-11-26T14:17:34,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:17:35,094 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,094 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(7752): Opening region: {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:17:35,094 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,095 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:17:35,095 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(7794): checking encryption for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,095 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(7797): checking classloading for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,097 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,107 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:35,108 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName A 2024-11-26T14:17:35,116 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:35,117 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:35,117 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,125 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:35,125 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName B 2024-11-26T14:17:35,125 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:35,128 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:35,128 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,131 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T14:17:35,131 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a053eb1b7cd48562bf65d3c9969f8a5b columnFamilyName C 2024-11-26T14:17:35,131 DEBUG [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:35,132 INFO [StoreOpener-a053eb1b7cd48562bf65d3c9969f8a5b-1 {}] regionserver.HStore(327): Store=a053eb1b7cd48562bf65d3c9969f8a5b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:17:35,132 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,132 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1038): replaying wal for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,133 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,134 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,134 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1048): stopping wal replay for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,134 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1060): Cleaning up temporary data for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,135 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:17:35,136 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1093): writing seq id for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,136 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1114): Opened a053eb1b7cd48562bf65d3c9969f8a5b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66308985, jitterRate=-0.01191912591457367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:17:35,136 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,137 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegion(1006): Region open journal for a053eb1b7cd48562bf65d3c9969f8a5b: Running coprocessor pre-open hook at 1732630655095Writing region info on filesystem at 1732630655095Initializing all the Stores at 1732630655097 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630655097Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630655097Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732630655097Cleaning up temporary data from old regions at 1732630655134 (+37 ms)Running coprocessor post-open hooks at 1732630655136 (+2 ms)Region opened successfully at 1732630655137 (+1 ms) 2024-11-26T14:17:35,137 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., pid=117, masterSystemTime=1732630655090 2024-11-26T14:17:35,139 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,139 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=117}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,139 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=OPEN, openSeqNum=5, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:17:35,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=115 2024-11-26T14:17:35,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 in 202 msec 2024-11-26T14:17:35,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-26T14:17:35,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, REOPEN/MOVE in 530 msec 2024-11-26T14:17:35,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T14:17:35,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 544 msec 2024-11-26T14:17:35,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 975 msec 2024-11-26T14:17:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T14:17:35,148 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:35,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,149 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d7c80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61496dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117a6f31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,153 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6664ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,154 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc6ab1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,156 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40aa5c90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,157 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d72e009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,158 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb2bfe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,160 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3defb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,161 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691b8dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,165 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1482d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3496adee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,168 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229b59c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,169 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e47097f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af95036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,172 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a673a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a15a0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f700a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,37705,-1 for getting cluster id 2024-11-26T14:17:35,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:17:35,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4854e9f7-b361-4b2a-9f10-47c2bb43584f' 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4854e9f7-b361-4b2a-9f10-47c2bb43584f" 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c660e38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,37705,-1] 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:17:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:35,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:17:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74936a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:17:35,178 DEBUG [Thread-2118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,178 DEBUG [Thread-2122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,178 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,178 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:35,178 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,179 DEBUG [Thread-2111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,179 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,179 DEBUG [Thread-2114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,179 DEBUG [Thread-2120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,180 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees 2024-11-26T14:17:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,181 DEBUG [Thread-2114 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,181 DEBUG [Thread-2114 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,181 DEBUG [Thread-2103 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,182 DEBUG [Thread-2109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:17:35,182 DEBUG [Thread-2103 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:35,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,184 DEBUG [Thread-2109 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,40025,1732630490995, seqNum=-1] 2024-11-26T14:17:35,184 DEBUG [Thread-2109 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:17:35,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=118, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:17:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,188 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=118, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:35,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5] 2024-11-26T14:17:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:35,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630715206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630715206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630655201, max timestamp=1732630655207Num uniques 859; 2024-11-26T14:17:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630655201, max timestamp=1732630655207Num uniques 859; 2024-11-26T14:17:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630655201, max timestamp=1732630655207Num uniques 859; 2024-11-26T14:17:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630715207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126313688aa7f1a489788a682d8eb674b04_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_2/A:col10/1732630655191/Put/seqid=0 2024-11-26T14:17:35,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742286_1462 (size=9714) 2024-11-26T14:17:35,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:35,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126313688aa7f1a489788a682d8eb674b04_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126313688aa7f1a489788a682d8eb674b04_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630715235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/801b11e043964a0f8fe8add0ac962ff8, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:35,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/801b11e043964a0f8fe8add0ac962ff8 is 175, key is test_row_2/A:col10/1732630655191/Put/seqid=0 2024-11-26T14:17:35,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630715238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742287_1463 (size=22359) 2024-11-26T14:17:35,251 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/801b11e043964a0f8fe8add0ac962ff8 2024-11-26T14:17:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:35,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4628e28d078e465683009423dde244de is 50, key is test_row_2/B:col10/1732630655191/Put/seqid=0 2024-11-26T14:17:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742288_1464 (size=7315) 2024-11-26T14:17:35,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630715313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630715313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630715313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=119 2024-11-26T14:17:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] handler.RSProcedureHandler(58): pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630715342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=119 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630715353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=119 2024-11-26T14:17:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=119}] handler.RSProcedureHandler(58): pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=119 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630715523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630715523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630715523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630715553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630715563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=119 2024-11-26T14:17:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] handler.RSProcedureHandler(58): pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=119 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=119 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:35,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4628e28d078e465683009423dde244de 2024-11-26T14:17:35,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c13fba2b74b543e482b2075cd2426974 is 50, key is test_row_2/C:col10/1732630655191/Put/seqid=0 2024-11-26T14:17:35,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742289_1465 (size=7315) 2024-11-26T14:17:35,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c13fba2b74b543e482b2075cd2426974 2024-11-26T14:17:35,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/801b11e043964a0f8fe8add0ac962ff8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8 2024-11-26T14:17:35,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8, entries=100, sequenceid=15, filesize=21.8 K 2024-11-26T14:17:35,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4628e28d078e465683009423dde244de as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de 2024-11-26T14:17:35,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de, entries=50, sequenceid=15, filesize=7.1 K 2024-11-26T14:17:35,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c13fba2b74b543e482b2075cd2426974 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974 2024-11-26T14:17:35,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974, entries=50, sequenceid=15, filesize=7.1 K 2024-11-26T14:17:35,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a053eb1b7cd48562bf65d3c9969f8a5b in 564ms, sequenceid=15, compaction requested=false 2024-11-26T14:17:35,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:35,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=119 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:35,804 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267d96438de62a4195bbef8a328161f3c1_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630655232/Put/seqid=0 2024-11-26T14:17:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742290_1466 (size=12154) 2024-11-26T14:17:35,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:35,844 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267d96438de62a4195bbef8a328161f3c1_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d96438de62a4195bbef8a328161f3c1_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:35,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c600921b83b2411f99a0cd30218652bd, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:35,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c600921b83b2411f99a0cd30218652bd is 175, key is test_row_0/A:col10/1732630655232/Put/seqid=0 2024-11-26T14:17:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630715849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630715850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630715852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630715863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630715873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742291_1467 (size=30955) 2024-11-26T14:17:35,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630715964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630715964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:35,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630715964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:35,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630716173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630716173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630716183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,284 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c600921b83b2411f99a0cd30218652bd 2024-11-26T14:17:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e3e843981f014cb6b3f18532d05f2c15 is 50, key is test_row_0/B:col10/1732630655232/Put/seqid=0 2024-11-26T14:17:36,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742292_1468 (size=12001) 2024-11-26T14:17:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630716384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630716384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630716483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630716483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630716493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:36,706 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e3e843981f014cb6b3f18532d05f2c15 2024-11-26T14:17:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/5dfcdef2f7264597897d8fd4be9a3632 is 50, key is test_row_0/C:col10/1732630655232/Put/seqid=0 2024-11-26T14:17:36,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742293_1469 (size=12001) 2024-11-26T14:17:36,772 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/5dfcdef2f7264597897d8fd4be9a3632 2024-11-26T14:17:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c600921b83b2411f99a0cd30218652bd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd 2024-11-26T14:17:36,781 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:17:36,783 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd, entries=150, sequenceid=49, filesize=30.2 K 2024-11-26T14:17:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e3e843981f014cb6b3f18532d05f2c15 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15 2024-11-26T14:17:36,809 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:17:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/5dfcdef2f7264597897d8fd4be9a3632 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632 2024-11-26T14:17:36,830 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T14:17:36,833 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for a053eb1b7cd48562bf65d3c9969f8a5b in 1029ms, sequenceid=49, compaction requested=false 2024-11-26T14:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=119}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=119 2024-11-26T14:17:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=119 2024-11-26T14:17:36,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-26T14:17:36,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6510 sec 2024-11-26T14:17:36,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=118, table=TestAcidGuarantees in 1.6640 sec 2024-11-26T14:17:36,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T14:17:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:37,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:37,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269d3e94eb830e4e0fa449d70c56c592dc_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742294_1470 (size=14594) 2024-11-26T14:17:37,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:37,055 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269d3e94eb830e4e0fa449d70c56c592dc_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d3e94eb830e4e0fa449d70c56c592dc_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:37,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/32bf4fe5af074404a1fe7ed641ed01e0, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:37,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/32bf4fe5af074404a1fe7ed641ed01e0 is 175, key is test_row_0/A:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630717070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630717073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630717081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742295_1471 (size=39549) 2024-11-26T14:17:37,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/32bf4fe5af074404a1fe7ed641ed01e0 2024-11-26T14:17:37,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/2f98dc4cb3204a748ece1ddb0090d4d3 is 50, key is test_row_0/B:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630717193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630717193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630717204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742296_1472 (size=12001) 2024-11-26T14:17:37,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/2f98dc4cb3204a748ece1ddb0090d4d3 2024-11-26T14:17:37,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8727ea79770e471f8d3ad07b4b5f2648 is 50, key is test_row_0/C:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742297_1473 (size=12001) 2024-11-26T14:17:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T14:17:37,335 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:37,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees 2024-11-26T14:17:37,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=120, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:37,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=120, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:37,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630717393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,397 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630717403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630717404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630717404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,408 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630717423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:37,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:37,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:37,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:37,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8727ea79770e471f8d3ad07b4b5f2648 2024-11-26T14:17:37,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/32bf4fe5af074404a1fe7ed641ed01e0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0 2024-11-26T14:17:37,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0, entries=200, sequenceid=62, filesize=38.6 K 2024-11-26T14:17:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/2f98dc4cb3204a748ece1ddb0090d4d3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3 2024-11-26T14:17:37,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-26T14:17:37,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8727ea79770e471f8d3ad07b4b5f2648 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648 2024-11-26T14:17:37,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630717715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630717715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648, entries=150, sequenceid=62, filesize=11.7 K 2024-11-26T14:17:37,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a053eb1b7cd48562bf65d3c9969f8a5b in 729ms, sequenceid=62, compaction requested=true 2024-11-26T14:17:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:37,728 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:37,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:37,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:37,729 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:37,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:37,730 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:37,730 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:37,730 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,730 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=90.7 K 2024-11-26T14:17:37,730 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,730 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0] 2024-11-26T14:17:37,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:37,730 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:37,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:37,731 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801b11e043964a0f8fe8add0ac962ff8, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630655190 2024-11-26T14:17:37,730 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:37,731 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,731 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=30.6 K 2024-11-26T14:17:37,732 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c600921b83b2411f99a0cd30218652bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630655232 2024-11-26T14:17:37,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:37,732 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4628e28d078e465683009423dde244de, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630655191 2024-11-26T14:17:37,733 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bf4fe5af074404a1fe7ed641ed01e0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655840 2024-11-26T14:17:37,733 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e843981f014cb6b3f18532d05f2c15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630655232 2024-11-26T14:17:37,734 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f98dc4cb3204a748ece1ddb0090d4d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655850 2024-11-26T14:17:37,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:37,753 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:37,754 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/92d597b2aec04cf7afab295270d1cdaf is 50, key is test_row_0/B:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630717792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,801 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:37,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f0a68064904d48c889ad1bd4cc732688_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630657736/Put/seqid=0 2024-11-26T14:17:37,888 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126e804b3ad7e374c55993102453c8fa648_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:37,890 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126e804b3ad7e374c55993102453c8fa648_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:37,891 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e804b3ad7e374c55993102453c8fa648_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:37,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742298_1474 (size=12104) 2024-11-26T14:17:37,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630717903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:37,922 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/92d597b2aec04cf7afab295270d1cdaf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92d597b2aec04cf7afab295270d1cdaf 2024-11-26T14:17:37,941 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 92d597b2aec04cf7afab295270d1cdaf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:37,941 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:37,941 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630657729; duration=0sec 2024-11-26T14:17:37,941 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:37,941 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:37,941 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:37,946 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:37,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742299_1475 (size=12154) 2024-11-26T14:17:37,946 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:37,946 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,947 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=30.6 K 2024-11-26T14:17:37,947 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c13fba2b74b543e482b2075cd2426974, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732630655191 2024-11-26T14:17:37,948 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfcdef2f7264597897d8fd4be9a3632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732630655232 2024-11-26T14:17:37,949 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8727ea79770e471f8d3ad07b4b5f2648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655850 2024-11-26T14:17:37,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742300_1476 (size=4469) 2024-11-26T14:17:37,954 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#399 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:37,954 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/429b007d508e4742a593b9a4dcce2924 is 175, key is test_row_0/A:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:37,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:37,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:37,978 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:37,979 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/aa54ee5f449f4082b113a77bf1ecd505 is 50, key is test_row_0/C:col10/1732630655850/Put/seqid=0 2024-11-26T14:17:38,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742301_1477 (size=31058) 2024-11-26T14:17:38,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742302_1478 (size=12104) 2024-11-26T14:17:38,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630718123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630718234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630718234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:38,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:38,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f0a68064904d48c889ad1bd4cc732688_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f0a68064904d48c889ad1bd4cc732688_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:38,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/10dc110ee9de48aa86ab88d6ca250826, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:38,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/10dc110ee9de48aa86ab88d6ca250826 is 175, key is test_row_0/A:col10/1732630657736/Put/seqid=0 2024-11-26T14:17:38,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742303_1479 (size=30955) 2024-11-26T14:17:38,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/10dc110ee9de48aa86ab88d6ca250826 2024-11-26T14:17:38,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/10fdf05cfa7e4bd3a4143cca437f8387 is 50, key is test_row_0/B:col10/1732630657736/Put/seqid=0 2024-11-26T14:17:38,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742304_1480 (size=12001) 2024-11-26T14:17:38,421 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/429b007d508e4742a593b9a4dcce2924 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924 2024-11-26T14:17:38,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/10fdf05cfa7e4bd3a4143cca437f8387 2024-11-26T14:17:38,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:38,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:38,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] handler.RSProcedureHandler(58): pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=121 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=121 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:38,430 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 429b007d508e4742a593b9a4dcce2924(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:38,430 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:38,430 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630657728; duration=0sec 2024-11-26T14:17:38,431 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:38,431 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:38,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/de66ca4d15e14ec99fa2592996bd78ca is 50, key is test_row_0/C:col10/1732630657736/Put/seqid=0 2024-11-26T14:17:38,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630718433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:38,457 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/aa54ee5f449f4082b113a77bf1ecd505 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/aa54ee5f449f4082b113a77bf1ecd505 2024-11-26T14:17:38,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742305_1481 (size=12001) 2024-11-26T14:17:38,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/de66ca4d15e14ec99fa2592996bd78ca 2024-11-26T14:17:38,462 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into aa54ee5f449f4082b113a77bf1ecd505(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:38,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:38,462 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=13, startTime=1732630657730; duration=0sec 2024-11-26T14:17:38,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:38,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:38,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/10dc110ee9de48aa86ab88d6ca250826 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826 2024-11-26T14:17:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:38,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826, entries=150, sequenceid=83, filesize=30.2 K 2024-11-26T14:17:38,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/10fdf05cfa7e4bd3a4143cca437f8387 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387 2024-11-26T14:17:38,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387, entries=150, sequenceid=83, filesize=11.7 K 2024-11-26T14:17:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/de66ca4d15e14ec99fa2592996bd78ca as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca 2024-11-26T14:17:38,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca, entries=150, sequenceid=83, filesize=11.7 K 2024-11-26T14:17:38,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a053eb1b7cd48562bf65d3c9969f8a5b in 751ms, sequenceid=83, compaction requested=false 2024-11-26T14:17:38,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:38,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:38,579 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269029205f11794ce7810d35acaf2fcfbd_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630657768/Put/seqid=0 2024-11-26T14:17:38,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742306_1482 (size=12154) 2024-11-26T14:17:38,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:38,653 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269029205f11794ce7810d35acaf2fcfbd_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269029205f11794ce7810d35acaf2fcfbd_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:38,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a923521291074ad997514d7f06d4bd44, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:38,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a923521291074ad997514d7f06d4bd44 is 175, key is test_row_0/A:col10/1732630657768/Put/seqid=0 2024-11-26T14:17:38,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742307_1483 (size=30955) 2024-11-26T14:17:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:39,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630659003, max timestamp=1732630659049Num uniques 859; 2024-11-26T14:17:39,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630659003, max timestamp=1732630659049Num uniques 859; 2024-11-26T14:17:39,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:39,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:39,105 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a923521291074ad997514d7f06d4bd44 2024-11-26T14:17:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/92ca841131b74baa8633622527e94b29 is 50, key is test_row_0/B:col10/1732630657768/Put/seqid=0 2024-11-26T14:17:39,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742308_1484 (size=12001) 2024-11-26T14:17:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630719107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630719223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630719242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,248 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630719253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,258 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630719402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,408 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630719433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630719436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,440 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:39,514 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/92ca841131b74baa8633622527e94b29 2024-11-26T14:17:39,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/b9900cdf286d47189d05e31909521e73 is 50, key is test_row_0/C:col10/1732630657768/Put/seqid=0 2024-11-26T14:17:39,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742309_1485 (size=12001) 2024-11-26T14:17:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630719743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:39,923 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/b9900cdf286d47189d05e31909521e73 2024-11-26T14:17:39,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a923521291074ad997514d7f06d4bd44 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44 2024-11-26T14:17:39,944 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44, entries=150, sequenceid=97, filesize=30.2 K 2024-11-26T14:17:39,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/92ca841131b74baa8633622527e94b29 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29 2024-11-26T14:17:39,971 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T14:17:39,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/b9900cdf286d47189d05e31909521e73 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73 2024-11-26T14:17:39,976 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T14:17:39,977 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for a053eb1b7cd48562bf65d3c9969f8a5b in 1398ms, sequenceid=97, compaction requested=true 2024-11-26T14:17:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=121 2024-11-26T14:17:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=121 2024-11-26T14:17:39,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees in 2.6480 sec 2024-11-26T14:17:39,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-26T14:17:39,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-26T14:17:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:40,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=129.71 KB heapSize=373.55 KB 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:40,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:40,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e76c8a8915e744f7b1a6e1e2c2b0c240_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742310_1486 (size=14594) 2024-11-26T14:17:40,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630720337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630720456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630720673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:40,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:40,754 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e76c8a8915e744f7b1a6e1e2c2b0c240_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e76c8a8915e744f7b1a6e1e2c2b0c240_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:40,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/30d7e070c0c0450b80125f0e5872234c, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:40,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/30d7e070c0c0450b80125f0e5872234c is 175, key is test_row_0/A:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742311_1487 (size=39549) 2024-11-26T14:17:40,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630720983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:41,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/30d7e070c0c0450b80125f0e5872234c 2024-11-26T14:17:41,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/aa51b36e4b9a47578de6c1a071363779 is 50, key is test_row_0/B:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742312_1488 (size=12001) 2024-11-26T14:17:41,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630721273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:41,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630721273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:41,280 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:41,280 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_2' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:41,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630721493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-26T14:17:41,518 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:41,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees 2024-11-26T14:17:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:41,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:41,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:41,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:41,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/aa51b36e4b9a47578de6c1a071363779 2024-11-26T14:17:41,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f920a74b2fdc45d5bf2743e89c1c67a7 is 50, key is test_row_0/C:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742313_1489 (size=12001) 2024-11-26T14:17:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:41,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-26T14:17:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-26T14:17:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:41,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-26T14:17:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:41,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:42,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f920a74b2fdc45d5bf2743e89c1c67a7 2024-11-26T14:17:42,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/30d7e070c0c0450b80125f0e5872234c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c 2024-11-26T14:17:42,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c, entries=200, sequenceid=128, filesize=38.6 K 2024-11-26T14:17:42,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/aa51b36e4b9a47578de6c1a071363779 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779 2024-11-26T14:17:42,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T14:17:42,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f920a74b2fdc45d5bf2743e89c1c67a7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7 2024-11-26T14:17:42,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T14:17:42,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~373.50 KB/382464, currentSize=46.96 KB/48090 for a053eb1b7cd48562bf65d3c9969f8a5b in 1763ms, sequenceid=128, compaction requested=true 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:42,032 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:42,032 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:42,033 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:42,033 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:42,033 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:42,034 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:42,034 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:42,034 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=129.4 K 2024-11-26T14:17:42,034 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92d597b2aec04cf7afab295270d1cdaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=47.0 K 2024-11-26T14:17:42,034 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c] 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d597b2aec04cf7afab295270d1cdaf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655850 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429b007d508e4742a593b9a4dcce2924, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655850 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 10fdf05cfa7e4bd3a4143cca437f8387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630657050 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10dc110ee9de48aa86ab88d6ca250826, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630657050 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ca841131b74baa8633622527e94b29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630657768 2024-11-26T14:17:42,034 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a923521291074ad997514d7f06d4bd44, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630657768 2024-11-26T14:17:42,035 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d7e070c0c0450b80125f0e5872234c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659066 2024-11-26T14:17:42,035 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa51b36e4b9a47578de6c1a071363779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659085 2024-11-26T14:17:42,040 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:42,042 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:42,043 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/99243259103148898439bfc25e6049c8 is 50, key is test_row_0/B:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:42,044 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268c7cbd2c0c6c4960a761bb543bec8a63_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:42,047 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268c7cbd2c0c6c4960a761bb543bec8a63_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:42,047 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268c7cbd2c0c6c4960a761bb543bec8a63_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:42,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742314_1490 (size=12241) 2024-11-26T14:17:42,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742315_1491 (size=4469) 2024-11-26T14:17:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:42,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-26T14:17:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:42,165 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:42,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f8d932e9aeb240008ff85d1bed9846a5_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630660275/Put/seqid=0 2024-11-26T14:17:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742316_1492 (size=12304) 2024-11-26T14:17:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:42,178 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f8d932e9aeb240008ff85d1bed9846a5_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8d932e9aeb240008ff85d1bed9846a5_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/f975c9ce2cf34615b9dc063d08eb4404, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:42,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/f975c9ce2cf34615b9dc063d08eb4404 is 175, key is test_row_0/A:col10/1732630660275/Put/seqid=0 2024-11-26T14:17:42,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742317_1493 (size=31105) 2024-11-26T14:17:42,452 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/99243259103148898439bfc25e6049c8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/99243259103148898439bfc25e6049c8 2024-11-26T14:17:42,455 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#411 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:42,455 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/7d8801b5571d4be3bd19c9f1e934a8bd is 175, key is test_row_0/A:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:42,460 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 99243259103148898439bfc25e6049c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:42,460 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:42,460 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=12, startTime=1732630662032; duration=0sec 2024-11-26T14:17:42,460 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:42,460 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:42,460 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:42,461 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:42,461 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:42,461 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:42,461 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/aa54ee5f449f4082b113a77bf1ecd505, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=47.0 K 2024-11-26T14:17:42,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting aa54ee5f449f4082b113a77bf1ecd505, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732630655850 2024-11-26T14:17:42,462 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting de66ca4d15e14ec99fa2592996bd78ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732630657050 2024-11-26T14:17:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742318_1494 (size=31195) 2024-11-26T14:17:42,463 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b9900cdf286d47189d05e31909521e73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732630657768 2024-11-26T14:17:42,463 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting f920a74b2fdc45d5bf2743e89c1c67a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659085 2024-11-26T14:17:42,470 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:42,471 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/65bd853cbc5247718471e05defbcf91a is 50, key is test_row_0/C:col10/1732630660267/Put/seqid=0 2024-11-26T14:17:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742319_1495 (size=12241) 2024-11-26T14:17:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:42,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:42,583 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/f975c9ce2cf34615b9dc063d08eb4404 2024-11-26T14:17:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c7a6ac2ea75845c39e2d5656aa035fa6 is 50, key is test_row_0/B:col10/1732630660275/Put/seqid=0 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630662561, max timestamp=1732630662598Num uniques 859; 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630662561, max timestamp=1732630662598Num uniques 859; 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630662561, max timestamp=1732630662598Num uniques 859; 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:42,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:42,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742320_1496 (size=12151) 2024-11-26T14:17:42,617 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c7a6ac2ea75845c39e2d5656aa035fa6 2024-11-26T14:17:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2efbdb1468a2481ba57f29ec9de3d22e is 50, key is test_row_0/C:col10/1732630660275/Put/seqid=0 2024-11-26T14:17:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742321_1497 (size=12151) 2024-11-26T14:17:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630722714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:42,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630722823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:42,870 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/7d8801b5571d4be3bd19c9f1e934a8bd as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd 2024-11-26T14:17:42,875 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 7d8801b5571d4be3bd19c9f1e934a8bd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:42,875 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:42,875 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=12, startTime=1732630662032; duration=0sec 2024-11-26T14:17:42,875 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:42,875 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:42,878 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/65bd853cbc5247718471e05defbcf91a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/65bd853cbc5247718471e05defbcf91a 2024-11-26T14:17:42,882 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 65bd853cbc5247718471e05defbcf91a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:42,882 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:42,882 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=12, startTime=1732630662032; duration=0sec 2024-11-26T14:17:42,882 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:42,882 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:43,034 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2efbdb1468a2481ba57f29ec9de3d22e 2024-11-26T14:17:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630723033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/f975c9ce2cf34615b9dc063d08eb4404 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404 2024-11-26T14:17:43,043 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404, entries=150, sequenceid=138, filesize=30.4 K 2024-11-26T14:17:43,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c7a6ac2ea75845c39e2d5656aa035fa6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6 2024-11-26T14:17:43,047 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6, entries=150, sequenceid=138, filesize=11.9 K 2024-11-26T14:17:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2efbdb1468a2481ba57f29ec9de3d22e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e 2024-11-26T14:17:43,052 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e, entries=150, sequenceid=138, filesize=11.9 K 2024-11-26T14:17:43,054 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a053eb1b7cd48562bf65d3c9969f8a5b in 888ms, sequenceid=138, compaction requested=false 2024-11-26T14:17:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:43,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=123 2024-11-26T14:17:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=123 2024-11-26T14:17:43,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-26T14:17:43,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-26T14:17:43,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees in 1.5400 sec 2024-11-26T14:17:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:43,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:43,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265264bd07cbca47d89c34294b1b13cf0b_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742322_1498 (size=14794) 2024-11-26T14:17:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630723388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630723433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,438 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630723463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,466 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630723493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-26T14:17:43,678 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:43,678 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees 2024-11-26T14:17:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:43,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:43,682 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:43,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:43,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630723705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:43,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:43,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265264bd07cbca47d89c34294b1b13cf0b_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265264bd07cbca47d89c34294b1b13cf0b_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:43,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0789cbaee5fa49ebaa8ae0ff4c958aa7, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:43,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 is 175, key is test_row_0/A:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:43,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742323_1499 (size=39749) 2024-11-26T14:17:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:43,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:43,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:43,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:43,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:43,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630724022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:44,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 2024-11-26T14:17:44,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/05c896f0d2bd43ed8126e04315b89f53 is 50, key is test_row_0/B:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742324_1500 (size=12151) 2024-11-26T14:17:44,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:44,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:44,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:44,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:44,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630724532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:44,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/05c896f0d2bd43ed8126e04315b89f53 2024-11-26T14:17:44,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:44,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f5a8dfc1a6ca40b7960ac08ff092ccba is 50, key is test_row_0/C:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:44,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742325_1501 (size=12151) 2024-11-26T14:17:44,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f5a8dfc1a6ca40b7960ac08ff092ccba 2024-11-26T14:17:44,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 2024-11-26T14:17:44,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7, entries=200, sequenceid=179, filesize=38.8 K 2024-11-26T14:17:44,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/05c896f0d2bd43ed8126e04315b89f53 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53 2024-11-26T14:17:44,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53, entries=150, sequenceid=179, filesize=11.9 K 2024-11-26T14:17:44,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/f5a8dfc1a6ca40b7960ac08ff092ccba as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba 2024-11-26T14:17:44,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba, entries=150, sequenceid=179, filesize=11.9 K 2024-11-26T14:17:44,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for a053eb1b7cd48562bf65d3c9969f8a5b in 1325ms, sequenceid=179, compaction requested=true 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:44,672 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:44,672 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:44,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:44,674 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:44,674 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:44,674 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:44,674 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:44,674 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,675 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,675 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/99243259103148898439bfc25e6049c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=35.7 K 2024-11-26T14:17:44,675 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=99.7 K 2024-11-26T14:17:44,675 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7] 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 99243259103148898439bfc25e6049c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659085 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8801b5571d4be3bd19c9f1e934a8bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659085 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f975c9ce2cf34615b9dc063d08eb4404, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732630660275 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c7a6ac2ea75845c39e2d5656aa035fa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732630660275 2024-11-26T14:17:44,675 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c896f0d2bd43ed8126e04315b89f53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662704 2024-11-26T14:17:44,676 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0789cbaee5fa49ebaa8ae0ff4c958aa7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662695 2024-11-26T14:17:44,685 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:44,686 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:44,686 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/612a332cb72345258db13f6552ea594c is 50, key is test_row_0/B:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:44,689 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264bfc7e52eae2453a97c4cd6d66fe1372_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:44,690 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264bfc7e52eae2453a97c4cd6d66fe1372_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:44,691 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264bfc7e52eae2453a97c4cd6d66fe1372_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:44,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742327_1503 (size=4469) 2024-11-26T14:17:44,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742326_1502 (size=12493) 2024-11-26T14:17:44,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-26T14:17:44,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:44,765 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T14:17:44,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eab04c8cafb54ed2ab217bf2b371e754_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630663370/Put/seqid=0 2024-11-26T14:17:44,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742328_1504 (size=12304) 2024-11-26T14:17:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:45,113 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#420 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:45,114 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/1e39fe81f2634913a22137d318473aaf is 175, key is test_row_0/A:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:45,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742329_1505 (size=31447) 2024-11-26T14:17:45,126 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/1e39fe81f2634913a22137d318473aaf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf 2024-11-26T14:17:45,129 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/612a332cb72345258db13f6552ea594c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/612a332cb72345258db13f6552ea594c 2024-11-26T14:17:45,134 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 1e39fe81f2634913a22137d318473aaf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:45,134 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:45,134 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630664672; duration=0sec 2024-11-26T14:17:45,134 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:45,134 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:45,134 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:45,138 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 612a332cb72345258db13f6552ea594c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:45,138 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:45,138 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630664672; duration=0sec 2024-11-26T14:17:45,138 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:45,138 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:45,140 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:45,141 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:45,141 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:45,141 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/65bd853cbc5247718471e05defbcf91a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=35.7 K 2024-11-26T14:17:45,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65bd853cbc5247718471e05defbcf91a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732630659085 2024-11-26T14:17:45,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2efbdb1468a2481ba57f29ec9de3d22e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732630660275 2024-11-26T14:17:45,142 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a8dfc1a6ca40b7960ac08ff092ccba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662704 2024-11-26T14:17:45,151 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:45,152 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/a0493fd696e74087b123100869cefbb6 is 50, key is test_row_0/C:col10/1732630663345/Put/seqid=0 2024-11-26T14:17:45,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:45,193 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eab04c8cafb54ed2ab217bf2b371e754_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eab04c8cafb54ed2ab217bf2b371e754_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:45,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742330_1506 (size=12493) 2024-11-26T14:17:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c480fa8a91e84ac68f2b70985c9eb8e3, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:45,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c480fa8a91e84ac68f2b70985c9eb8e3 is 175, key is test_row_0/A:col10/1732630663370/Put/seqid=0 2024-11-26T14:17:45,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742331_1507 (size=31105) 2024-11-26T14:17:45,222 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c480fa8a91e84ac68f2b70985c9eb8e3 2024-11-26T14:17:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e1431153ffd94a4d9ad8a29a97bf5667 is 50, key is test_row_0/B:col10/1732630663370/Put/seqid=0 2024-11-26T14:17:45,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742332_1508 (size=12151) 2024-11-26T14:17:45,246 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e1431153ffd94a4d9ad8a29a97bf5667 2024-11-26T14:17:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/77307cbe3d9f43118fc16e1b0ccf4d1c is 50, key is test_row_0/C:col10/1732630663370/Put/seqid=0 2024-11-26T14:17:45,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742333_1509 (size=12151) 2024-11-26T14:17:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:45,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630665333, max timestamp=1732630665348Num uniques 859; 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630665333, max timestamp=1732630665348Num uniques 859; 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630665333, max timestamp=1732630665348Num uniques 859; 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:45,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630665362, max timestamp=1732630665396Num uniques 1018; 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630665362, max timestamp=1732630665396Num uniques 1018; 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630665362, max timestamp=1732630665396Num uniques 1018; 2024-11-26T14:17:45,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630665432, max timestamp=1732630665451Num uniques 1027; 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630665432, max timestamp=1732630665451Num uniques 1027; 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732630665432, max timestamp=1732630665451Num uniques 1027; 2024-11-26T14:17:45,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T14:17:45,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630725539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630725540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:45,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630725543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:45,548 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:45,601 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/a0493fd696e74087b123100869cefbb6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/a0493fd696e74087b123100869cefbb6 2024-11-26T14:17:45,609 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into a0493fd696e74087b123100869cefbb6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:45,609 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:45,609 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=13, startTime=1732630664672; duration=0sec 2024-11-26T14:17:45,609 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:45,609 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:45,656 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/77307cbe3d9f43118fc16e1b0ccf4d1c 2024-11-26T14:17:45,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630725653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:45,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630725654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:45,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c480fa8a91e84ac68f2b70985c9eb8e3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3 2024-11-26T14:17:45,670 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3, entries=150, sequenceid=188, filesize=30.4 K 2024-11-26T14:17:45,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/e1431153ffd94a4d9ad8a29a97bf5667 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667 2024-11-26T14:17:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667, entries=150, sequenceid=188, filesize=11.9 K 2024-11-26T14:17:45,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/77307cbe3d9f43118fc16e1b0ccf4d1c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c 2024-11-26T14:17:45,682 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c, entries=150, sequenceid=188, filesize=11.9 K 2024-11-26T14:17:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a053eb1b7cd48562bf65d3c9969f8a5b in 918ms, sequenceid=188, compaction requested=false 2024-11-26T14:17:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:45,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=125 2024-11-26T14:17:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=125 2024-11-26T14:17:45,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-26T14:17:45,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0030 sec 2024-11-26T14:17:45,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees in 2.0120 sec 2024-11-26T14:17:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-26T14:17:45,825 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:45,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees 2024-11-26T14:17:45,828 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:45,831 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:45,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:45,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-11-26T14:17:45,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630665515, max timestamp=1732630665872Num uniques 1086; 2024-11-26T14:17:45,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:17:45,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630665515, max timestamp=1732630665872Num uniques 1086; 2024-11-26T14:17:45,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:17:45,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732630665515, max timestamp=1732630665872Num uniques 1086; 2024-11-26T14:17:45,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T14:17:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:45,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:45,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263fa548066526422b9dfc57c8819cd5d1_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:45,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742334_1510 (size=12454) 2024-11-26T14:17:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:45,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:45,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:45,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630665927, max timestamp=1732630666003Num uniques 859; 2024-11-26T14:17:46,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:46,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630665927, max timestamp=1732630666003Num uniques 859; 2024-11-26T14:17:46,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:17:46,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732630665927, max timestamp=1732630666003Num uniques 859; 2024-11-26T14:17:46,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T14:17:46,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630666051, max timestamp=1732630666095Num uniques 1018; 2024-11-26T14:17:46,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T14:17:46,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732630666051, max timestamp=1732630666095Num uniques 1009; 2024-11-26T14:17:46,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-26T14:17:46,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732630666051, max timestamp=1732630666095Num uniques 1009; 2024-11-26T14:17:46,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T14:17:46,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:46,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1732630666143, max timestamp=1732630666191Num uniques 1077; 2024-11-26T14:17:46,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=8092, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T14:17:46,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732630666143, max timestamp=1732630666191Num uniques 1077; 2024-11-26T14:17:46,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732630666143, max timestamp=1732630666191Num uniques 1027; 2024-11-26T14:17:46,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T14:17:46,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-26T14:17:46,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,318 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:46,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263fa548066526422b9dfc57c8819cd5d1_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fa548066526422b9dfc57c8819cd5d1_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:46,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/3462fed406f0456c88dd06553f52bd4d, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/3462fed406f0456c88dd06553f52bd4d is 175, key is test_row_0/A:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742335_1511 (size=31255) 2024-11-26T14:17:46,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/3462fed406f0456c88dd06553f52bd4d 2024-11-26T14:17:46,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/d18a0324889040bab0d17834e615b583 is 50, key is test_row_0/B:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630666327, max timestamp=1732630666359Num uniques 1036; 2024-11-26T14:17:46,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630666327, max timestamp=1732630666359Num uniques 1036; 2024-11-26T14:17:46,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T14:17:46,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T14:17:46,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732630666327, max timestamp=1732630666359Num uniques 1036; 2024-11-26T14:17:46,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T14:17:46,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742336_1512 (size=12301) 2024-11-26T14:17:46,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/d18a0324889040bab0d17834e615b583 2024-11-26T14:17:46,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/4483652a2ea04000a75c223d612f1064 is 50, key is test_row_0/C:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742337_1513 (size=12301) 2024-11-26T14:17:46,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/4483652a2ea04000a75c223d612f1064 2024-11-26T14:17:46,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/3462fed406f0456c88dd06553f52bd4d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d 2024-11-26T14:17:46,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d, entries=150, sequenceid=264, filesize=30.5 K 2024-11-26T14:17:46,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/d18a0324889040bab0d17834e615b583 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583 2024-11-26T14:17:46,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583, entries=150, sequenceid=264, filesize=12.0 K 2024-11-26T14:17:46,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/4483652a2ea04000a75c223d612f1064 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064 2024-11-26T14:17:46,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064, entries=150, sequenceid=264, filesize=12.0 K 2024-11-26T14:17:46,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~83.58 KB/85584, currentSize=140.89 KB/144270 for a053eb1b7cd48562bf65d3c9969f8a5b in 566ms, sequenceid=264, compaction requested=true 2024-11-26T14:17:46,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:46,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:46,442 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:46,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:46,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:46,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:46,443 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:46,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:46,443 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:46,443 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:46,443 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,443 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,444 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=91.6 K 2024-11-26T14:17:46,444 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/612a332cb72345258db13f6552ea594c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.1 K 2024-11-26T14:17:46,444 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,444 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d] 2024-11-26T14:17:46,444 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 612a332cb72345258db13f6552ea594c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662704 2024-11-26T14:17:46,444 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e39fe81f2634913a22137d318473aaf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662704 2024-11-26T14:17:46,444 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting e1431153ffd94a4d9ad8a29a97bf5667, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630663362 2024-11-26T14:17:46,444 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d18a0324889040bab0d17834e615b583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:46,445 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c480fa8a91e84ac68f2b70985c9eb8e3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630663362 2024-11-26T14:17:46,445 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3462fed406f0456c88dd06553f52bd4d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:46,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=147.60 KB heapSize=418.31 KB 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:46,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:46,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:46,465 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,470 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:46,470 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/ac769e1f94364d83a792586e57e08e57 is 50, key is test_row_0/B:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,478 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126b81898b9903847b4ba2fed55e97efb50_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,480 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126b81898b9903847b4ba2fed55e97efb50_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,480 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b81898b9903847b4ba2fed55e97efb50_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630726493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630726503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112649de8aa30f434953bca4a9bd2abfccd4_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630666444/Put/seqid=0 2024-11-26T14:17:46,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742338_1514 (size=12745) 2024-11-26T14:17:46,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742339_1515 (size=4469) 2024-11-26T14:17:46,546 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/ac769e1f94364d83a792586e57e08e57 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ac769e1f94364d83a792586e57e08e57 2024-11-26T14:17:46,547 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#428 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:46,548 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6b998a8f859a4f2684652a5892e21cb6 is 175, key is test_row_0/A:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,562 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into ac769e1f94364d83a792586e57e08e57(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:46,562 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:46,562 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630666442; duration=0sec 2024-11-26T14:17:46,562 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:46,562 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:46,562 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:46,564 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:46,564 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:46,564 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,564 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/a0493fd696e74087b123100869cefbb6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.1 K 2024-11-26T14:17:46,564 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting a0493fd696e74087b123100869cefbb6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732630662704 2024-11-26T14:17:46,566 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 77307cbe3d9f43118fc16e1b0ccf4d1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732630663362 2024-11-26T14:17:46,567 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4483652a2ea04000a75c223d612f1064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:46,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742340_1516 (size=14994) 2024-11-26T14:17:46,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:46,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112649de8aa30f434953bca4a9bd2abfccd4_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112649de8aa30f434953bca4a9bd2abfccd4_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:46,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ca6799c6d67b4c259c0beac32ea7827f, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:46,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ca6799c6d67b4c259c0beac32ea7827f is 175, key is test_row_0/A:col10/1732630666444/Put/seqid=0 2024-11-26T14:17:46,599 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:46,599 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/25306b9761a4462baea5145adf4cc13f is 50, key is test_row_0/C:col10/1732630665537/Put/seqid=0 2024-11-26T14:17:46,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742341_1517 (size=31699) 2024-11-26T14:17:46,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,608 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6b998a8f859a4f2684652a5892e21cb6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6 2024-11-26T14:17:46,615 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 6b998a8f859a4f2684652a5892e21cb6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:46,615 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:46,615 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630666442; duration=0sec 2024-11-26T14:17:46,615 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:46,615 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:46,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742342_1518 (size=39949) 2024-11-26T14:17:46,618 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ca6799c6d67b4c259c0beac32ea7827f 2024-11-26T14:17:46,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630726617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630726617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/13bc365575ec449c9e4ed05dfc856ace is 50, key is test_row_0/B:col10/1732630666444/Put/seqid=0 2024-11-26T14:17:46,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742343_1519 (size=12745) 2024-11-26T14:17:46,655 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/25306b9761a4462baea5145adf4cc13f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/25306b9761a4462baea5145adf4cc13f 2024-11-26T14:17:46,677 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 25306b9761a4462baea5145adf4cc13f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:46,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:46,677 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=13, startTime=1732630666443; duration=0sec 2024-11-26T14:17:46,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:46,677 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:46,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742344_1520 (size=12301) 2024-11-26T14:17:46,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/13bc365575ec449c9e4ed05dfc856ace 2024-11-26T14:17:46,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/0ff63a771a9d40a0a131fe0dea10b7ab is 50, key is test_row_0/C:col10/1732630666444/Put/seqid=0 2024-11-26T14:17:46,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742345_1521 (size=12301) 2024-11-26T14:17:46,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630726833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630726833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:46,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:46,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:46,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:47,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:47,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630727143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630727143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/0ff63a771a9d40a0a131fe0dea10b7ab 2024-11-26T14:17:47,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ca6799c6d67b4c259c0beac32ea7827f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f 2024-11-26T14:17:47,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f, entries=200, sequenceid=353, filesize=39.0 K 2024-11-26T14:17:47,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/13bc365575ec449c9e4ed05dfc856ace as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace 2024-11-26T14:17:47,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace, entries=150, sequenceid=353, filesize=12.0 K 2024-11-26T14:17:47,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/0ff63a771a9d40a0a131fe0dea10b7ab as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab 2024-11-26T14:17:47,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab, entries=150, sequenceid=353, filesize=12.0 K 2024-11-26T14:17:47,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~418.27 KB/428304, currentSize=33.54 KB/34350 for a053eb1b7cd48562bf65d3c9969f8a5b in 748ms, sequenceid=353, compaction requested=false 2024-11-26T14:17:47,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:47,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-26T14:17:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:47,254 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:47,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263d34075f66d7481abab36fc89b2dd537_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630666493/Put/seqid=0 2024-11-26T14:17:47,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742346_1522 (size=12454) 2024-11-26T14:17:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:47,325 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263d34075f66d7481abab36fc89b2dd537_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d34075f66d7481abab36fc89b2dd537_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:47,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a7fc324716d94685884ffdeb54f1fdc6, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:47,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a7fc324716d94685884ffdeb54f1fdc6 is 175, key is test_row_0/A:col10/1732630666493/Put/seqid=0 2024-11-26T14:17:47,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742347_1523 (size=31255) 2024-11-26T14:17:47,347 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a7fc324716d94685884ffdeb54f1fdc6 2024-11-26T14:17:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cf8adf1938c04b72b75a193254fd9aaa is 50, key is test_row_0/B:col10/1732630666493/Put/seqid=0 2024-11-26T14:17:47,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742348_1524 (size=12301) 2024-11-26T14:17:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:47,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630667638, max timestamp=1732630667669Num uniques 859; 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630667638, max timestamp=1732630667669Num uniques 859; 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732630667638, max timestamp=1732630667669Num uniques 859; 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:47,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T14:17:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630727709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630727710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630727715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,774 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cf8adf1938c04b72b75a193254fd9aaa 2024-11-26T14:17:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/531147ce3d734761abf71ba3da34c3a0 is 50, key is test_row_0/C:col10/1732630666493/Put/seqid=0 2024-11-26T14:17:47,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742349_1525 (size=12301) 2024-11-26T14:17:47,797 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/531147ce3d734761abf71ba3da34c3a0 2024-11-26T14:17:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a7fc324716d94685884ffdeb54f1fdc6 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6 2024-11-26T14:17:47,806 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6, entries=150, sequenceid=364, filesize=30.5 K 2024-11-26T14:17:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cf8adf1938c04b72b75a193254fd9aaa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa 2024-11-26T14:17:47,811 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa, entries=150, sequenceid=364, filesize=12.0 K 2024-11-26T14:17:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/531147ce3d734761abf71ba3da34c3a0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0 2024-11-26T14:17:47,820 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0, entries=150, sequenceid=364, filesize=12.0 K 2024-11-26T14:17:47,821 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for a053eb1b7cd48562bf65d3c9969f8a5b in 567ms, sequenceid=364, compaction requested=true 2024-11-26T14:17:47,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:47,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:47,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=127 2024-11-26T14:17:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=127 2024-11-26T14:17:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:47,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-11-26T14:17:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:47,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T14:17:47,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-26T14:17:47,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-11-26T14:17:47,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees in 2.0030 sec 2024-11-26T14:17:47,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630727833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630727838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630727839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269c5ce2e58e784099a9d9101e7e2f792e_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:47,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742350_1526 (size=14994) 2024-11-26T14:17:47,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:47,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269c5ce2e58e784099a9d9101e7e2f792e_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c5ce2e58e784099a9d9101e7e2f792e_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:47,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c64695de939548f4b63ec10b9a2b827a, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:47,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c64695de939548f4b63ec10b9a2b827a is 175, key is test_row_0/A:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:47,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742351_1527 (size=39949) 2024-11-26T14:17:47,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c64695de939548f4b63ec10b9a2b827a 2024-11-26T14:17:47,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b9e82eb06f6d4c10ac473db37db10c5a is 50, key is test_row_0/B:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742352_1528 (size=12301) 2024-11-26T14:17:47,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b9e82eb06f6d4c10ac473db37db10c5a 2024-11-26T14:17:47,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3931fca3d62c4d07a718d24385c3b48b is 50, key is test_row_0/C:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742353_1529 (size=12301) 2024-11-26T14:17:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630727943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630727953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630727953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-26T14:17:47,983 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:47,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees 2024-11-26T14:17:47,985 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:47,985 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:47,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:48,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-26T14:17:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630728153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630728162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630728162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-26T14:17:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:48,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3931fca3d62c4d07a718d24385c3b48b 2024-11-26T14:17:48,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/c64695de939548f4b63ec10b9a2b827a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a 2024-11-26T14:17:48,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a, entries=200, sequenceid=402, filesize=39.0 K 2024-11-26T14:17:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b9e82eb06f6d4c10ac473db37db10c5a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a 2024-11-26T14:17:48,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a, entries=150, sequenceid=402, filesize=12.0 K 2024-11-26T14:17:48,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3931fca3d62c4d07a718d24385c3b48b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b 2024-11-26T14:17:48,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b, entries=150, sequenceid=402, filesize=12.0 K 2024-11-26T14:17:48,386 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:17:48,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for a053eb1b7cd48562bf65d3c9969f8a5b in 563ms, sequenceid=402, compaction requested=true 2024-11-26T14:17:48,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:48,389 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:48,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:48,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:48,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:48,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:48,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:48,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:48,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:48,390 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:48,390 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:48,390 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,390 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=139.5 K 2024-11-26T14:17:48,390 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,390 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a] 2024-11-26T14:17:48,392 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b998a8f859a4f2684652a5892e21cb6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:48,392 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6799c6d67b4c259c0beac32ea7827f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732630666416 2024-11-26T14:17:48,392 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:48,392 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:48,393 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,393 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ac769e1f94364d83a792586e57e08e57, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=48.5 K 2024-11-26T14:17:48,393 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7fc324716d94685884ffdeb54f1fdc6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732630666449 2024-11-26T14:17:48,393 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64695de939548f4b63ec10b9a2b827a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:48,393 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ac769e1f94364d83a792586e57e08e57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:48,393 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bc365575ec449c9e4ed05dfc856ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732630666416 2024-11-26T14:17:48,394 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8adf1938c04b72b75a193254fd9aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732630666449 2024-11-26T14:17:48,394 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e82eb06f6d4c10ac473db37db10c5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:48,425 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:48,437 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#441 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:48,438 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c57fa37290e3411ea978301aec7985b7 is 50, key is test_row_0/B:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:48,441 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411262ae43a7822114ebb99ce92c205fa87bb_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:48,443 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411262ae43a7822114ebb99ce92c205fa87bb_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:48,444 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262ae43a7822114ebb99ce92c205fa87bb_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:48,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-26T14:17:48,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,444 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742354_1530 (size=4469) 2024-11-26T14:17:48,469 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#440 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:48,469 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0daa5aae7e76465aa6bf801b22bcf26f is 175, key is test_row_0/A:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:48,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ffed7d9ce0344298bcc0a8849b768da6_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630667835/Put/seqid=0 2024-11-26T14:17:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:48,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:48,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742355_1531 (size=12881) 2024-11-26T14:17:48,500 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c57fa37290e3411ea978301aec7985b7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c57fa37290e3411ea978301aec7985b7 2024-11-26T14:17:48,505 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into c57fa37290e3411ea978301aec7985b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:48,505 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:48,505 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=12, startTime=1732630668389; duration=0sec 2024-11-26T14:17:48,505 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:48,505 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:48,505 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:48,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742357_1533 (size=12454) 2024-11-26T14:17:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742356_1532 (size=31835) 2024-11-26T14:17:48,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:48,513 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:48,514 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:48,514 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/25306b9761a4462baea5145adf4cc13f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=48.5 K 2024-11-26T14:17:48,514 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 25306b9761a4462baea5145adf4cc13f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732630665537 2024-11-26T14:17:48,515 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff63a771a9d40a0a131fe0dea10b7ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732630666416 2024-11-26T14:17:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:48,516 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 531147ce3d734761abf71ba3da34c3a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732630666449 2024-11-26T14:17:48,516 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3931fca3d62c4d07a718d24385c3b48b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:48,519 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/0daa5aae7e76465aa6bf801b22bcf26f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f 2024-11-26T14:17:48,520 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ffed7d9ce0344298bcc0a8849b768da6_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ffed7d9ce0344298bcc0a8849b768da6_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/fac2493da37346908a01b00663f118a9, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/fac2493da37346908a01b00663f118a9 is 175, key is test_row_0/A:col10/1732630667835/Put/seqid=0 2024-11-26T14:17:48,527 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 0daa5aae7e76465aa6bf801b22bcf26f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:48,527 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:48,527 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=12, startTime=1732630668389; duration=0sec 2024-11-26T14:17:48,527 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:48,527 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:48,530 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#443 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:48,530 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/bd5a196fe8fb48c9975483a1e71327f9 is 50, key is test_row_0/C:col10/1732630667705/Put/seqid=0 2024-11-26T14:17:48,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732630668520, max timestamp=1732630668533Num uniques 859; 2024-11-26T14:17:48,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T14:17:48,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742358_1534 (size=31255) 2024-11-26T14:17:48,552 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/fac2493da37346908a01b00663f118a9 2024-11-26T14:17:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742359_1535 (size=12881) 2024-11-26T14:17:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/0fbc5e9277cb419ca0b770b3629a8daa is 50, key is test_row_0/B:col10/1732630667835/Put/seqid=0 2024-11-26T14:17:48,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742360_1536 (size=12301) 2024-11-26T14:17:48,583 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/bd5a196fe8fb48c9975483a1e71327f9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/bd5a196fe8fb48c9975483a1e71327f9 2024-11-26T14:17:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630728582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630728587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630728587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,598 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into bd5a196fe8fb48c9975483a1e71327f9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:48,598 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:48,599 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=12, startTime=1732630668390; duration=0sec 2024-11-26T14:17:48,599 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:48,599 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630728694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630728693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630728703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630728902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630728902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630728913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:48,973 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/0fbc5e9277cb419ca0b770b3629a8daa 2024-11-26T14:17:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d4173ad42be249cb94f6848570a0908f is 50, key is test_row_0/C:col10/1732630667835/Put/seqid=0 2024-11-26T14:17:48,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742361_1537 (size=12301) 2024-11-26T14:17:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630729213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630729214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630729223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,384 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d4173ad42be249cb94f6848570a0908f 2024-11-26T14:17:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/fac2493da37346908a01b00663f118a9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9 2024-11-26T14:17:49,390 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9, entries=150, sequenceid=409, filesize=30.5 K 2024-11-26T14:17:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/0fbc5e9277cb419ca0b770b3629a8daa as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa 2024-11-26T14:17:49,393 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa, entries=150, sequenceid=409, filesize=12.0 K 2024-11-26T14:17:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d4173ad42be249cb94f6848570a0908f as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f 2024-11-26T14:17:49,397 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-26T14:17:49,397 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=158.78 KB/162590 for a053eb1b7cd48562bf65d3c9969f8a5b in 953ms, sequenceid=409, compaction requested=false 2024-11-26T14:17:49,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:49,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:49,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-11-26T14:17:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=129 2024-11-26T14:17:49,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-26T14:17:49,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-26T14:17:49,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees in 1.4170 sec 2024-11-26T14:17:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:49,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=172.20 KB heapSize=506.25 KB 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:49,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f76f0f5fd60b418d8773a3f305f10b96_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630729727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742362_1538 (size=12454) 2024-11-26T14:17:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630729732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630729733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630729843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630729843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:50,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630730052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630730053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-26T14:17:50,134 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:50,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees 2024-11-26T14:17:50,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:50,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-26T14:17:50,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:50,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:50,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f76f0f5fd60b418d8773a3f305f10b96_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f76f0f5fd60b418d8773a3f305f10b96_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:50,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/16b5f4a5fab3442b801cfcd19c4d4eaf, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/16b5f4a5fab3442b801cfcd19c4d4eaf is 175, key is test_row_0/A:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:50,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742363_1539 (size=31255) 2024-11-26T14:17:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-26T14:17:50,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-26T14:17:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630730363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630730363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:50,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-26T14:17:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-26T14:17:50,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/16b5f4a5fab3442b801cfcd19c4d4eaf 2024-11-26T14:17:50,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/df2de83b5ad14a99933a883c9cfd8a15 is 50, key is test_row_0/B:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:50,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742364_1540 (size=12301) 2024-11-26T14:17:50,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/df2de83b5ad14a99933a883c9cfd8a15 2024-11-26T14:17:50,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/abae2f3b96624bab80ea2713bb3cea04 is 50, key is test_row_0/C:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:50,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742365_1541 (size=12301) 2024-11-26T14:17:50,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/abae2f3b96624bab80ea2713bb3cea04 2024-11-26T14:17:50,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/16b5f4a5fab3442b801cfcd19c4d4eaf as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf 2024-11-26T14:17:50,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf, entries=150, sequenceid=445, filesize=30.5 K 2024-11-26T14:17:50,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/df2de83b5ad14a99933a883c9cfd8a15 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15 2024-11-26T14:17:50,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-26T14:17:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:50,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:50,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T14:17:50,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/abae2f3b96624bab80ea2713bb3cea04 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04 2024-11-26T14:17:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04, entries=150, sequenceid=445, filesize=12.0 K 2024-11-26T14:17:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~172.20 KB/176330, heapSize ~506.20 KB/518352, currentSize=6.71 KB/6870 for a053eb1b7cd48562bf65d3c9969f8a5b in 872ms, sequenceid=445, compaction requested=true 2024-11-26T14:17:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:50,600 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:50,600 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:50,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:50,602 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c57fa37290e3411ea978301aec7985b7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.6 K 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:50,602 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=92.1 K 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c57fa37290e3411ea978301aec7985b7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,602 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf] 2024-11-26T14:17:50,602 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbc5e9277cb419ca0b770b3629a8daa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732630667831 2024-11-26T14:17:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0daa5aae7e76465aa6bf801b22bcf26f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting df2de83b5ad14a99933a883c9cfd8a15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732630668547 2024-11-26T14:17:50,603 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac2493da37346908a01b00663f118a9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732630667831 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b5f4a5fab3442b801cfcd19c4d4eaf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732630668547 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,608 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,610 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112677cd34afdb674a37addfd2487d334890_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,610 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,610 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6e2ea561352e4fcb901e1539e0865731 is 50, key is test_row_0/B:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,611 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112677cd34afdb674a37addfd2487d334890_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112677cd34afdb674a37addfd2487d334890_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742366_1542 (size=12983) 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742367_1543 (size=4469) 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,753 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:50,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266b2c5f0b6c384acd96de4c71fcb34282_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_1/A:col10/1732630669731/Put/seqid=0 2024-11-26T14:17:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-26T14:17:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742368_1544 (size=7374) 2024-11-26T14:17:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,813 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266b2c5f0b6c384acd96de4c71fcb34282_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266b2c5f0b6c384acd96de4c71fcb34282_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9ad6870544e24b778cae1d6508a428ad, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9ad6870544e24b778cae1d6508a428ad is 175, key is test_row_1/A:col10/1732630669731/Put/seqid=0 2024-11-26T14:17:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:50,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742369_1545 (size=13865) 2024-11-26T14:17:50,847 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9ad6870544e24b778cae1d6508a428ad 2024-11-26T14:17:50,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/82cc88ce393340d994471281a3cd06ce is 50, key is test_row_1/B:col10/1732630669731/Put/seqid=0 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630670865, max timestamp=1732630670893Num uniques 859; 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630670865, max timestamp=1732630670893Num uniques 859; 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732630670865, max timestamp=1732630670893Num uniques 859; 2024-11-26T14:17:50,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/19945b63c2b4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T14:17:50,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742370_1546 (size=7415) 2024-11-26T14:17:50,910 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/82cc88ce393340d994471281a3cd06ce 2024-11-26T14:17:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3abf68076abc40268240d2fedcbd4290 is 50, key is test_row_1/C:col10/1732630669731/Put/seqid=0 2024-11-26T14:17:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742371_1547 (size=7415) 2024-11-26T14:17:50,928 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3abf68076abc40268240d2fedcbd4290 2024-11-26T14:17:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9ad6870544e24b778cae1d6508a428ad as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad 2024-11-26T14:17:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad, entries=50, sequenceid=449, filesize=13.5 K 2024-11-26T14:17:50,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/82cc88ce393340d994471281a3cd06ce as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce 2024-11-26T14:17:50,940 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce, entries=50, sequenceid=449, filesize=7.2 K 2024-11-26T14:17:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3abf68076abc40268240d2fedcbd4290 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290 2024-11-26T14:17:50,944 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290, entries=50, sequenceid=449, filesize=7.2 K 2024-11-26T14:17:50,948 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=60.38 KB/61830 for a053eb1b7cd48562bf65d3c9969f8a5b in 195ms, sequenceid=449, compaction requested=true 2024-11-26T14:17:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-11-26T14:17:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:50,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-26T14:17:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=131 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T14:17:50,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-26T14:17:50,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-11-26T14:17:50,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees in 821 msec 2024-11-26T14:17:50,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260a3f2361889b418c86f21ad15d37fe54_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:50,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742372_1548 (size=12454) 2024-11-26T14:17:50,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:50,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260a3f2361889b418c86f21ad15d37fe54_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a3f2361889b418c86f21ad15d37fe54_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:50,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/2121f920c3d548b2858de45d7e5398f0, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:50,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/2121f920c3d548b2858de45d7e5398f0 is 175, key is test_row_0/A:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:50,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742373_1549 (size=31255) 2024-11-26T14:17:51,001 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/2121f920c3d548b2858de45d7e5398f0 2024-11-26T14:17:51,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6bd05e2d385043278b4bf37f42249cd8 is 50, key is test_row_0/B:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742374_1550 (size=12301) 2024-11-26T14:17:51,018 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#450 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:51,019 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/74de777b79c24993b31adf0e495ea392 is 175, key is test_row_0/A:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:51,020 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6e2ea561352e4fcb901e1539e0865731 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6e2ea561352e4fcb901e1539e0865731 2024-11-26T14:17:51,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742375_1551 (size=31937) 2024-11-26T14:17:51,048 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 6e2ea561352e4fcb901e1539e0865731(size=12.7 K), total size for store is 19.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:51,048 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,048 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630670600; duration=0sec 2024-11-26T14:17:51,049 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:51,049 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:51,049 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:51,052 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:51,052 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:51,052 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,052 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/bd5a196fe8fb48c9975483a1e71327f9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=43.8 K 2024-11-26T14:17:51,053 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5a196fe8fb48c9975483a1e71327f9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732630667705 2024-11-26T14:17:51,053 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting d4173ad42be249cb94f6848570a0908f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732630667831 2024-11-26T14:17:51,053 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting abae2f3b96624bab80ea2713bb3cea04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732630668547 2024-11-26T14:17:51,054 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 3abf68076abc40268240d2fedcbd4290, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732630669731 2024-11-26T14:17:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630731050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630731051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,061 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:51,061 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/594d6a738adb43f9b0050a0e1aa66396 is 50, key is test_row_0/C:col10/1732630668547/Put/seqid=0 2024-11-26T14:17:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630731062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742376_1552 (size=13017) 2024-11-26T14:17:51,091 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/594d6a738adb43f9b0050a0e1aa66396 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/594d6a738adb43f9b0050a0e1aa66396 2024-11-26T14:17:51,099 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 594d6a738adb43f9b0050a0e1aa66396(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:51,099 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,099 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=12, startTime=1732630670600; duration=0sec 2024-11-26T14:17:51,099 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:51,099 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630731163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630731173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630731173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-26T14:17:51,273 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:51,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-11-26T14:17:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T14:17:51,275 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:51,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:51,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:51,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630731372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T14:17:51,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630731383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630731383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6bd05e2d385043278b4bf37f42249cd8 2024-11-26T14:17:51,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-26T14:17:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:51,433 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/74de777b79c24993b31adf0e495ea392 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392 2024-11-26T14:17:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:51,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8eeaf08ff9b049028c1e927a152d6a91 is 50, key is test_row_0/C:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:51,440 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 74de777b79c24993b31adf0e495ea392(size=31.2 K), total size for store is 44.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:51,440 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,440 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630670600; duration=0sec 2024-11-26T14:17:51,440 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:51,440 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:51,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742377_1553 (size=12301) 2024-11-26T14:17:51,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8eeaf08ff9b049028c1e927a152d6a91 2024-11-26T14:17:51,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/2121f920c3d548b2858de45d7e5398f0 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0 2024-11-26T14:17:51,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0, entries=150, sequenceid=478, filesize=30.5 K 2024-11-26T14:17:51,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6bd05e2d385043278b4bf37f42249cd8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8 2024-11-26T14:17:51,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8, entries=150, sequenceid=478, filesize=12.0 K 2024-11-26T14:17:51,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/8eeaf08ff9b049028c1e927a152d6a91 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91 2024-11-26T14:17:51,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91, entries=150, sequenceid=478, filesize=12.0 K 2024-11-26T14:17:51,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~225.56 KB/230976, currentSize=100.63 KB/103050 for a053eb1b7cd48562bf65d3c9969f8a5b in 529ms, sequenceid=478, compaction requested=true 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:51,478 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:51,478 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:51,480 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:51,480 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 77057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:51,481 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,481 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,481 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=75.3 K 2024-11-26T14:17:51,481 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6e2ea561352e4fcb901e1539e0865731, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=31.9 K 2024-11-26T14:17:51,481 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0] 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2ea561352e4fcb901e1539e0865731, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732630668547 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74de777b79c24993b31adf0e495ea392, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732630668547 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cc88ce393340d994471281a3cd06ce, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732630669731 2024-11-26T14:17:51,481 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad6870544e24b778cae1d6508a428ad, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732630669731 2024-11-26T14:17:51,482 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd05e2d385043278b4bf37f42249cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732630670911 2024-11-26T14:17:51,482 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2121f920c3d548b2858de45d7e5398f0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732630670911 2024-11-26T14:17:51,500 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,509 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:51,510 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/bdbc17e0ae4c413686cb09e365939906 is 50, key is test_row_0/B:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:51,522 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a73048fc286a4aeab1143f4ccc1df551_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,524 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a73048fc286a4aeab1143f4ccc1df551_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,524 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a73048fc286a4aeab1143f4ccc1df551_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742378_1554 (size=13085) 2024-11-26T14:17:51,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742379_1555 (size=4469) 2024-11-26T14:17:51,548 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#458 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:51,548 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ab12e6eece4c4c8eac6feb996d2e3480 is 175, key is test_row_0/A:col10/1732630670919/Put/seqid=0 2024-11-26T14:17:51,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742380_1556 (size=32039) 2024-11-26T14:17:51,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,586 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:51,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T14:17:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666dace0704424f608bdf0b85f0943249_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630671058/Put/seqid=0 2024-11-26T14:17:51,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742381_1557 (size=12454) 2024-11-26T14:17:51,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:51,640 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112666dace0704424f608bdf0b85f0943249_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666dace0704424f608bdf0b85f0943249_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:51,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/b82e49dfdc604562a3e932b21617fad5, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/b82e49dfdc604562a3e932b21617fad5 is 175, key is test_row_0/A:col10/1732630671058/Put/seqid=0 2024-11-26T14:17:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742382_1558 (size=31255) 2024-11-26T14:17:51,666 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=499, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/b82e49dfdc604562a3e932b21617fad5 2024-11-26T14:17:51,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c696c80a5d604760a702afffe2e8a154 is 50, key is test_row_0/B:col10/1732630671058/Put/seqid=0 2024-11-26T14:17:51,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742383_1559 (size=12301) 2024-11-26T14:17:51,677 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c696c80a5d604760a702afffe2e8a154 2024-11-26T14:17:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/6cc164e8069948ccbc36467f374b92b2 is 50, key is test_row_0/C:col10/1732630671058/Put/seqid=0 2024-11-26T14:17:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:51,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742384_1560 (size=12301) 2024-11-26T14:17:51,687 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/6cc164e8069948ccbc36467f374b92b2 2024-11-26T14:17:51,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/b82e49dfdc604562a3e932b21617fad5 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5 2024-11-26T14:17:51,699 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5, entries=150, sequenceid=499, filesize=30.5 K 2024-11-26T14:17:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/c696c80a5d604760a702afffe2e8a154 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154 2024-11-26T14:17:51,703 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154, entries=150, sequenceid=499, filesize=12.0 K 2024-11-26T14:17:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/6cc164e8069948ccbc36467f374b92b2 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2 2024-11-26T14:17:51,709 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2, entries=150, sequenceid=499, filesize=12.0 K 2024-11-26T14:17:51,709 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=60.38 KB/61830 for a053eb1b7cd48562bf65d3c9969f8a5b in 123ms, sequenceid=499, compaction requested=true 2024-11-26T14:17:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-11-26T14:17:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:51,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T14:17:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=133 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:51,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-26T14:17:51,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 435 msec 2024-11-26T14:17:51,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 441 msec 2024-11-26T14:17:51,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126df723422e81f4e9191f874554fe224e9_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742385_1561 (size=14994) 2024-11-26T14:17:51,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:51,730 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126df723422e81f4e9191f874554fe224e9_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126df723422e81f4e9191f874554fe224e9_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:51,731 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a12a47c4c2fc497f887119a7d4f9f22e, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:51,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a12a47c4c2fc497f887119a7d4f9f22e is 175, key is test_row_0/A:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:51,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742386_1562 (size=39949) 2024-11-26T14:17:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630731748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630731749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630731749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630731863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630731863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630731863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T14:17:51,903 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:51,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-11-26T14:17:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:51,905 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:51,906 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:51,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:51,951 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/bdbc17e0ae4c413686cb09e365939906 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/bdbc17e0ae4c413686cb09e365939906 2024-11-26T14:17:51,956 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into bdbc17e0ae4c413686cb09e365939906(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:51,956 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,956 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630671478; duration=0sec 2024-11-26T14:17:51,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:51,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:51,957 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:51,959 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:51,959 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:51,959 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:51,959 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/594d6a738adb43f9b0050a0e1aa66396, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.7 K 2024-11-26T14:17:51,960 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 594d6a738adb43f9b0050a0e1aa66396, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732630668547 2024-11-26T14:17:51,962 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eeaf08ff9b049028c1e927a152d6a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732630670911 2024-11-26T14:17:51,962 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc164e8069948ccbc36467f374b92b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732630670992 2024-11-26T14:17:51,963 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/ab12e6eece4c4c8eac6feb996d2e3480 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480 2024-11-26T14:17:51,967 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into ab12e6eece4c4c8eac6feb996d2e3480(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:51,967 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:51,967 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630671478; duration=0sec 2024-11-26T14:17:51,967 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:51,967 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:51,985 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:51,985 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3c5c173bb10f45f9ad812f50bd0f9d8e is 50, key is test_row_0/C:col10/1732630671058/Put/seqid=0 2024-11-26T14:17:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742387_1563 (size=13119) 2024-11-26T14:17:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:52,021 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3c5c173bb10f45f9ad812f50bd0f9d8e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3c5c173bb10f45f9ad812f50bd0f9d8e 2024-11-26T14:17:52,036 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 3c5c173bb10f45f9ad812f50bd0f9d8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:52,036 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:52,036 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=13, startTime=1732630671478; duration=0sec 2024-11-26T14:17:52,036 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:52,036 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:52,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-26T14:17:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630732073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630732074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630732076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a12a47c4c2fc497f887119a7d4f9f22e 2024-11-26T14:17:52,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cc521e3b73044c65abe82fe9e1c726e1 is 50, key is test_row_0/B:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:52,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742388_1564 (size=12301) 2024-11-26T14:17:52,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-26T14:17:52,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:52,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:52,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-26T14:17:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630732382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630732382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630732393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-26T14:17:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:52,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cc521e3b73044c65abe82fe9e1c726e1 2024-11-26T14:17:52,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3849510335e24455b3189995db52908b is 50, key is test_row_0/C:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742389_1565 (size=12301) 2024-11-26T14:17:52,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3849510335e24455b3189995db52908b 2024-11-26T14:17:52,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/a12a47c4c2fc497f887119a7d4f9f22e as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e 2024-11-26T14:17:52,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e, entries=200, sequenceid=512, filesize=39.0 K 2024-11-26T14:17:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/cc521e3b73044c65abe82fe9e1c726e1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1 2024-11-26T14:17:52,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1, entries=150, sequenceid=512, filesize=12.0 K 2024-11-26T14:17:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/3849510335e24455b3189995db52908b as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b 2024-11-26T14:17:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b, entries=150, sequenceid=512, filesize=12.0 K 2024-11-26T14:17:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a053eb1b7cd48562bf65d3c9969f8a5b in 919ms, sequenceid=512, compaction requested=true 2024-11-26T14:17:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:52,629 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:52,630 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:52,633 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:52,633 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:52,633 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:52,633 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,633 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/bdbc17e0ae4c413686cb09e365939906, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.8 K 2024-11-26T14:17:52,633 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:52,633 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,633 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=100.8 K 2024-11-26T14:17:52,633 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,633 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e] 2024-11-26T14:17:52,634 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting bdbc17e0ae4c413686cb09e365939906, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732630670911 2024-11-26T14:17:52,634 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting c696c80a5d604760a702afffe2e8a154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732630670992 2024-11-26T14:17:52,635 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting cc521e3b73044c65abe82fe9e1c726e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732630671699 2024-11-26T14:17:52,635 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab12e6eece4c4c8eac6feb996d2e3480, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732630670911 2024-11-26T14:17:52,635 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82e49dfdc604562a3e932b21617fad5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732630670992 2024-11-26T14:17:52,635 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12a47c4c2fc497f887119a7d4f9f22e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732630671699 2024-11-26T14:17:52,648 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:52,648 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/805e5aa397d14366847bad6d8aa788e9 is 50, key is test_row_0/B:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:52,658 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:52,661 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268d0c11118f704e1c857748c53fbdfa32_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:52,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742390_1566 (size=13187) 2024-11-26T14:17:52,663 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268d0c11118f704e1c857748c53fbdfa32_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:52,664 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268d0c11118f704e1c857748c53fbdfa32_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:52,668 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/805e5aa397d14366847bad6d8aa788e9 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/805e5aa397d14366847bad6d8aa788e9 2024-11-26T14:17:52,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:52,677 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:52,682 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 805e5aa397d14366847bad6d8aa788e9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:52,682 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:52,682 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630672629; duration=0sec 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. because compaction request was cancelled 2024-11-26T14:17:52,683 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:52,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b277c0a6ee7740ab8aca1c244f794351_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630671748/Put/seqid=0 2024-11-26T14:17:52,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742392_1568 (size=12454) 2024-11-26T14:17:52,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742391_1567 (size=4469) 2024-11-26T14:17:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:52,701 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#468 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:52,701 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/bef66f8fac2a4e6b9143bca6f79cc984 is 175, key is test_row_0/A:col10/1732630671699/Put/seqid=0 2024-11-26T14:17:52,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742393_1569 (size=32141) 2024-11-26T14:17:52,706 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b277c0a6ee7740ab8aca1c244f794351_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b277c0a6ee7740ab8aca1c244f794351_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9a09f232e78d48ae886a07acc4cfde39, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9a09f232e78d48ae886a07acc4cfde39 is 175, key is test_row_0/A:col10/1732630671748/Put/seqid=0 2024-11-26T14:17:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742394_1570 (size=31255) 2024-11-26T14:17:52,725 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=535, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9a09f232e78d48ae886a07acc4cfde39 2024-11-26T14:17:52,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/60d8696689004c609dcd042f4f9dc819 is 50, key is test_row_0/B:col10/1732630671748/Put/seqid=0 2024-11-26T14:17:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742395_1571 (size=12301) 2024-11-26T14:17:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:52,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630732926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630732926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630732927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630733033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:53,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630733043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630733043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,110 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/bef66f8fac2a4e6b9143bca6f79cc984 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984 2024-11-26T14:17:53,114 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into bef66f8fac2a4e6b9143bca6f79cc984(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:53,114 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:53,114 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630672629; duration=0sec 2024-11-26T14:17:53,114 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:53,114 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:53,153 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/60d8696689004c609dcd042f4f9dc819 2024-11-26T14:17:53,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c57b77d9f2384ef998c4e69ec6d4e894 is 50, key is test_row_0/C:col10/1732630671748/Put/seqid=0 2024-11-26T14:17:53,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742396_1572 (size=12301) 2024-11-26T14:17:53,166 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c57b77d9f2384ef998c4e69ec6d4e894 2024-11-26T14:17:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9a09f232e78d48ae886a07acc4cfde39 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39 2024-11-26T14:17:53,176 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39, entries=150, sequenceid=535, filesize=30.5 K 2024-11-26T14:17:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/60d8696689004c609dcd042f4f9dc819 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819 2024-11-26T14:17:53,183 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819, entries=150, sequenceid=535, filesize=12.0 K 2024-11-26T14:17:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/c57b77d9f2384ef998c4e69ec6d4e894 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894 2024-11-26T14:17:53,187 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894, entries=150, sequenceid=535, filesize=12.0 K 2024-11-26T14:17:53,188 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for a053eb1b7cd48562bf65d3c9969f8a5b in 511ms, sequenceid=535, compaction requested=true 2024-11-26T14:17:53,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:53,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:53,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-11-26T14:17:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-11-26T14:17:53,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-26T14:17:53,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-26T14:17:53,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.2900 sec 2024-11-26T14:17:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:53,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265c209e5e90ac436799533919097a79d3_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742397_1573 (size=14994) 2024-11-26T14:17:53,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:53,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265c209e5e90ac436799533919097a79d3_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c209e5e90ac436799533919097a79d3_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:53,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/465ca332c9f1462a8089890b4ab742f1, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:53,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/465ca332c9f1462a8089890b4ab742f1 is 175, key is test_row_0/A:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742398_1574 (size=39949) 2024-11-26T14:17:53,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=552, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/465ca332c9f1462a8089890b4ab742f1 2024-11-26T14:17:53,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4c35d887011f4dce8e716a919722e687 is 50, key is test_row_0/B:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742399_1575 (size=12301) 2024-11-26T14:17:53,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4c35d887011f4dce8e716a919722e687 2024-11-26T14:17:53,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2cb240349cda4b6099686dcc12a9a234 is 50, key is test_row_0/C:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742400_1576 (size=12301) 2024-11-26T14:17:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630733301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630733301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630733307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630733413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630733413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630733423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42634 deadline: 1732630733493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42610 deadline: 1732630733493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,497 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_0' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,497 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 19945b63c2b4,40025,1732630490995 for 'test_row_1' in a053eb1b7cd48562bf65d3c9969f8a5b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630733623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630733623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630733633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2cb240349cda4b6099686dcc12a9a234 2024-11-26T14:17:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/465ca332c9f1462a8089890b4ab742f1 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1 2024-11-26T14:17:53,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1, entries=200, sequenceid=552, filesize=39.0 K 2024-11-26T14:17:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/4c35d887011f4dce8e716a919722e687 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687 2024-11-26T14:17:53,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687, entries=150, sequenceid=552, filesize=12.0 K 2024-11-26T14:17:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/2cb240349cda4b6099686dcc12a9a234 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234 2024-11-26T14:17:53,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234, entries=150, sequenceid=552, filesize=12.0 K 2024-11-26T14:17:53,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a053eb1b7cd48562bf65d3c9969f8a5b in 451ms, sequenceid=552, compaction requested=true 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:53,696 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:53,696 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:53,697 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:53,697 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:53,697 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/805e5aa397d14366847bad6d8aa788e9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=36.9 K 2024-11-26T14:17:53,697 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=100.9 K 2024-11-26T14:17:53,697 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1] 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 805e5aa397d14366847bad6d8aa788e9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732630671699 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting bef66f8fac2a4e6b9143bca6f79cc984, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732630671699 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 60d8696689004c609dcd042f4f9dc819, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1732630671734 2024-11-26T14:17:53,697 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a09f232e78d48ae886a07acc4cfde39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1732630671734 2024-11-26T14:17:53,698 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c35d887011f4dce8e716a919722e687, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:53,698 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465ca332c9f1462a8089890b4ab742f1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:53,703 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:53,705 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:53,705 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126625ed19a5af848b3b1139f3e18789b4e_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:53,705 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/9ec23905ee1a4dbcaf31a28b59389899 is 50, key is test_row_0/B:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,706 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126625ed19a5af848b3b1139f3e18789b4e_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:53,707 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126625ed19a5af848b3b1139f3e18789b4e_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:53,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742402_1578 (size=4469) 2024-11-26T14:17:53,711 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#476 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:53,711 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/7585530e57a24cfbb90fac8106b892c3 is 175, key is test_row_0/A:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742401_1577 (size=13289) 2024-11-26T14:17:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742403_1579 (size=32243) 2024-11-26T14:17:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:53,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:53,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112690cc8ebc4c744df8b459dd02ec0451df_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630673935/Put/seqid=0 2024-11-26T14:17:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742404_1580 (size=12454) 2024-11-26T14:17:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630733954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630733955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630733959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-26T14:17:54,053 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:54,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T14:17:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-26T14:17:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:54,054 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T14:17:54,055 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T14:17:54,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T14:17:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630734063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630734063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630734073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,118 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/9ec23905ee1a4dbcaf31a28b59389899 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/9ec23905ee1a4dbcaf31a28b59389899 2024-11-26T14:17:54,119 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/7585530e57a24cfbb90fac8106b892c3 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3 2024-11-26T14:17:54,123 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 9ec23905ee1a4dbcaf31a28b59389899(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:54,123 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 7585530e57a24cfbb90fac8106b892c3(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:54,124 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=13, startTime=1732630673696; duration=0sec 2024-11-26T14:17:54,124 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=13, startTime=1732630673696; duration=0sec 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:54,124 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:54,125 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:54,125 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:54,125 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,125 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3c5c173bb10f45f9ad812f50bd0f9d8e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=48.8 K 2024-11-26T14:17:54,126 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5c173bb10f45f9ad812f50bd0f9d8e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732630670992 2024-11-26T14:17:54,126 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3849510335e24455b3189995db52908b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732630671699 2024-11-26T14:17:54,126 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting c57b77d9f2384ef998c4e69ec6d4e894, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1732630671734 2024-11-26T14:17:54,127 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb240349cda4b6099686dcc12a9a234, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:54,135 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:54,136 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/4438b53dcf0b4d248eeab40547b84d8a is 50, key is test_row_0/C:col10/1732630673244/Put/seqid=0 2024-11-26T14:17:54,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742405_1581 (size=13255) 2024-11-26T14:17:54,144 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/4438b53dcf0b4d248eeab40547b84d8a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4438b53dcf0b4d248eeab40547b84d8a 2024-11-26T14:17:54,148 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 4438b53dcf0b4d248eeab40547b84d8a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:54,148 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:54,149 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=12, startTime=1732630673696; duration=0sec 2024-11-26T14:17:54,149 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:54,149 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:17:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:54,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-26T14:17:54,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:54,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630734273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630734273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630734282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:54,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112690cc8ebc4c744df8b459dd02ec0451df_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112690cc8ebc4c744df8b459dd02ec0451df_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:54,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/775e5f1ffc824aa690880e7eb1abe736, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:54,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/775e5f1ffc824aa690880e7eb1abe736 is 175, key is test_row_0/A:col10/1732630673935/Put/seqid=0 2024-11-26T14:17:54,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742406_1582 (size=31255) 2024-11-26T14:17:54,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-26T14:17:54,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:54,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-26T14:17:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630734583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630734583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630734593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:54,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-26T14:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:17:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:54,751 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=570, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/775e5f1ffc824aa690880e7eb1abe736 2024-11-26T14:17:54,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/ef548808f208477a96cbe1176dee4e49 is 50, key is test_row_0/B:col10/1732630673935/Put/seqid=0 2024-11-26T14:17:54,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742407_1583 (size=12301) 2024-11-26T14:17:54,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/ef548808f208477a96cbe1176dee4e49 2024-11-26T14:17:54,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/1e0a55ce3c344866bf4d5be7d552ae2d is 50, key is test_row_0/C:col10/1732630673935/Put/seqid=0 2024-11-26T14:17:54,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742408_1584 (size=12301) 2024-11-26T14:17:54,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/1e0a55ce3c344866bf4d5be7d552ae2d 2024-11-26T14:17:54,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/775e5f1ffc824aa690880e7eb1abe736 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736 2024-11-26T14:17:54,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736, entries=150, sequenceid=570, filesize=30.5 K 2024-11-26T14:17:54,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/ef548808f208477a96cbe1176dee4e49 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49 2024-11-26T14:17:54,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49, entries=150, sequenceid=570, filesize=12.0 K 2024-11-26T14:17:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/1e0a55ce3c344866bf4d5be7d552ae2d as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d 2024-11-26T14:17:54,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d, entries=150, sequenceid=570, filesize=12.0 K 2024-11-26T14:17:54,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a053eb1b7cd48562bf65d3c9969f8a5b in 868ms, sequenceid=570, compaction requested=false 2024-11-26T14:17:54,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:54,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40025 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:54,818 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263e68d1f8fe20497bbcceee3dd866e688_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630673958/Put/seqid=0 2024-11-26T14:17:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742409_1585 (size=12454) 2024-11-26T14:17:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. as already flushing 2024-11-26T14:17:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630735123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630735124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630735125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,181 INFO [Thread-2114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2114. 2024-11-26T14:17:55,181 DEBUG [Thread-2114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:55,181 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,181 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:55,184 INFO [Thread-2120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2120. 2024-11-26T14:17:55,184 DEBUG [Thread-2120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:55,184 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,184 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:55,184 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-11-26T14:17:55,184 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:55,184 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,184 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:55,185 INFO [Thread-2118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2118. 2024-11-26T14:17:55,185 DEBUG [Thread-2118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:55,185 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,185 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:55,187 INFO [Thread-2122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2122. 2024-11-26T14:17:55,187 DEBUG [Thread-2122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:55,187 DEBUG [Thread-2122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,188 DEBUG [Thread-2122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:55,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:55,230 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263e68d1f8fe20497bbcceee3dd866e688_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263e68d1f8fe20497bbcceee3dd866e688_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:55,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6519efacd300490fae70852a327234cb, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6519efacd300490fae70852a327234cb is 175, key is test_row_0/A:col10/1732630673958/Put/seqid=0 2024-11-26T14:17:55,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630735233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630735233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742410_1586 (size=31255) 2024-11-26T14:17:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630735236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630735444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630735444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630735444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,636 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=587, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6519efacd300490fae70852a327234cb 2024-11-26T14:17:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/12e36f732dd14c92b1d84c806a287670 is 50, key is test_row_0/B:col10/1732630673958/Put/seqid=0 2024-11-26T14:17:55,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742411_1587 (size=12301) 2024-11-26T14:17:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630735753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630735753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630735754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:55,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:56,046 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/12e36f732dd14c92b1d84c806a287670 2024-11-26T14:17:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/107afb74dca44baf961751ed51a00b7c is 50, key is test_row_0/C:col10/1732630673958/Put/seqid=0 2024-11-26T14:17:56,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742412_1588 (size=12301) 2024-11-26T14:17:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42568 deadline: 1732630736264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42650 deadline: 1732630736263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42640 deadline: 1732630736263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 2024-11-26T14:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 , the old value is region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T14:17:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., hostname=19945b63c2b4,40025,1732630490995, seqNum=5 because the exception is null or not the one we care about 2024-11-26T14:17:56,454 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/107afb74dca44baf961751ed51a00b7c 2024-11-26T14:17:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/6519efacd300490fae70852a327234cb as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb 2024-11-26T14:17:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb, entries=150, sequenceid=587, filesize=30.5 K 2024-11-26T14:17:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/12e36f732dd14c92b1d84c806a287670 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670 2024-11-26T14:17:56,467 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670, entries=150, sequenceid=587, filesize=12.0 K 2024-11-26T14:17:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/107afb74dca44baf961751ed51a00b7c as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c 2024-11-26T14:17:56,472 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c, entries=150, sequenceid=587, filesize=12.0 K 2024-11-26T14:17:56,473 INFO [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a053eb1b7cd48562bf65d3c9969f8a5b in 1655ms, sequenceid=587, compaction requested=true 2024-11-26T14:17:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:56,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-26T14:17:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-26T14:17:56,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-26T14:17:56,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4190 sec 2024-11-26T14:17:56,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.4250 sec 2024-11-26T14:17:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] regionserver.HRegion(8855): Flush requested on a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:57,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T14:17:57,276 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-26T14:17:57,276 INFO [Thread-2109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2109. 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:17:57,276 DEBUG [Thread-2109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:57,276 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,276 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:17:57,276 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:17:57,276 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:17:57,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:57,276 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:57,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f1cd51b1cc564fa5a20503549ffa9fc9_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:57,284 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-11-26T14:17:57,284 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:17:57,284 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,284 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:17:57,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:17:57,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742413_1589 (size=12454) 2024-11-26T14:17:57,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:17:57,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f1cd51b1cc564fa5a20503549ffa9fc9_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1cd51b1cc564fa5a20503549ffa9fc9_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:17:57,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/395e8fc2c379450daa30cb9f2bf4ad59, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:57,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/395e8fc2c379450daa30cb9f2bf4ad59 is 175, key is test_row_0/A:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742414_1590 (size=31255) 2024-11-26T14:17:57,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=607, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/395e8fc2c379450daa30cb9f2bf4ad59 2024-11-26T14:17:57,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6897c2423873418c8c9850374575b360 is 50, key is test_row_0/B:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:57,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742415_1591 (size=12301) 2024-11-26T14:17:58,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6897c2423873418c8c9850374575b360 2024-11-26T14:17:58,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/186406512dfe4a3db729374e5736c588 is 50, key is test_row_0/C:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:58,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742416_1592 (size=12301) 2024-11-26T14:17:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-26T14:17:58,223 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:17:58,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/186406512dfe4a3db729374e5736c588 2024-11-26T14:17:58,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/395e8fc2c379450daa30cb9f2bf4ad59 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59 2024-11-26T14:17:58,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59, entries=150, sequenceid=607, filesize=30.5 K 2024-11-26T14:17:58,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/6897c2423873418c8c9850374575b360 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360 2024-11-26T14:17:58,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360, entries=150, sequenceid=607, filesize=12.0 K 2024-11-26T14:17:58,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/186406512dfe4a3db729374e5736c588 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588 2024-11-26T14:17:58,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588, entries=150, sequenceid=607, filesize=12.0 K 2024-11-26T14:17:58,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for a053eb1b7cd48562bf65d3c9969f8a5b in 1271ms, sequenceid=607, compaction requested=true 2024-11-26T14:17:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T14:17:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T14:17:58,547 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:58,548 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a053eb1b7cd48562bf65d3c9969f8a5b:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T14:17:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 126008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/A is initiating minor compaction (all files) 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:58,549 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/A in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/B is initiating minor compaction (all files) 2024-11-26T14:17:58,549 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/B in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:58,549 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=123.1 K 2024-11-26T14:17:58,549 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/9ec23905ee1a4dbcaf31a28b59389899, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=49.0 K 2024-11-26T14:17:58,549 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. files: [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59] 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec23905ee1a4dbcaf31a28b59389899, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7585530e57a24cfbb90fac8106b892c3, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:58,549 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting ef548808f208477a96cbe1176dee4e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732630673264 2024-11-26T14:17:58,550 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775e5f1ffc824aa690880e7eb1abe736, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732630673264 2024-11-26T14:17:58,550 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e36f732dd14c92b1d84c806a287670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630673952 2024-11-26T14:17:58,550 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6519efacd300490fae70852a327234cb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630673952 2024-11-26T14:17:58,550 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 6897c2423873418c8c9850374575b360, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732630675124 2024-11-26T14:17:58,550 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] compactions.Compactor(225): Compacting 395e8fc2c379450daa30cb9f2bf4ad59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732630675124 2024-11-26T14:17:58,558 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:58,559 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#B#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:58,559 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/972941db3cc04831a4f626e836ef6b06 is 50, key is test_row_0/B:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:58,559 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126039907b8e1ba4ea88e5957baf16d04f4_a053eb1b7cd48562bf65d3c9969f8a5b store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:58,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742417_1593 (size=13425) 2024-11-26T14:17:58,564 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126039907b8e1ba4ea88e5957baf16d04f4_a053eb1b7cd48562bf65d3c9969f8a5b, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:58,564 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126039907b8e1ba4ea88e5957baf16d04f4_a053eb1b7cd48562bf65d3c9969f8a5b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:17:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742418_1594 (size=4469) 2024-11-26T14:17:58,966 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/972941db3cc04831a4f626e836ef6b06 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/972941db3cc04831a4f626e836ef6b06 2024-11-26T14:17:58,968 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#A#compaction#488 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:58,968 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/62fdf575bae04ddda1237932a85b0396 is 175, key is test_row_0/A:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:58,970 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/B of a053eb1b7cd48562bf65d3c9969f8a5b into 972941db3cc04831a4f626e836ef6b06(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:58,970 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:58,970 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/B, priority=12, startTime=1732630678547; duration=0sec 2024-11-26T14:17:58,970 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T14:17:58,970 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:B 2024-11-26T14:17:58,970 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:17:58,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742419_1595 (size=32379) 2024-11-26T14:17:58,972 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T14:17:58,972 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1541): a053eb1b7cd48562bf65d3c9969f8a5b/C is initiating minor compaction (all files) 2024-11-26T14:17:58,972 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a053eb1b7cd48562bf65d3c9969f8a5b/C in TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:17:58,972 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4438b53dcf0b4d248eeab40547b84d8a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588] into tmpdir=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp, totalSize=49.0 K 2024-11-26T14:17:58,972 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 4438b53dcf0b4d248eeab40547b84d8a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1732630672903 2024-11-26T14:17:58,972 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0a55ce3c344866bf4d5be7d552ae2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732630673264 2024-11-26T14:17:58,973 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 107afb74dca44baf961751ed51a00b7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=587, earliestPutTs=1732630673952 2024-11-26T14:17:58,973 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] compactions.Compactor(225): Compacting 186406512dfe4a3db729374e5736c588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732630675124 2024-11-26T14:17:58,979 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a053eb1b7cd48562bf65d3c9969f8a5b#C#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:17:58,979 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/9b8d63c111e14fa8887a2f4d875b24ed is 50, key is test_row_0/C:col10/1732630677274/Put/seqid=0 2024-11-26T14:17:58,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742420_1596 (size=13391) 2024-11-26T14:17:59,378 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/62fdf575bae04ddda1237932a85b0396 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/62fdf575bae04ddda1237932a85b0396 2024-11-26T14:17:59,384 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/A of a053eb1b7cd48562bf65d3c9969f8a5b into 62fdf575bae04ddda1237932a85b0396(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:59,384 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:59,384 INFO [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/A, priority=12, startTime=1732630678547; duration=0sec 2024-11-26T14:17:59,384 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:59,384 DEBUG [RS:0;19945b63c2b4:40025-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:A 2024-11-26T14:17:59,386 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/9b8d63c111e14fa8887a2f4d875b24ed as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/9b8d63c111e14fa8887a2f4d875b24ed 2024-11-26T14:17:59,390 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a053eb1b7cd48562bf65d3c9969f8a5b/C of a053eb1b7cd48562bf65d3c9969f8a5b into 9b8d63c111e14fa8887a2f4d875b24ed(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:17:59,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a053eb1b7cd48562bf65d3c9969f8a5b: 2024-11-26T14:17:59,390 INFO [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., storeName=a053eb1b7cd48562bf65d3c9969f8a5b/C, priority=12, startTime=1732630678548; duration=0sec 2024-11-26T14:17:59,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T14:17:59,390 DEBUG [RS:0;19945b63c2b4:40025-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a053eb1b7cd48562bf65d3c9969f8a5b:C 2024-11-26T14:18:03,545 INFO [Thread-2111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2111. 2024-11-26T14:18:03,545 DEBUG [Thread-2111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:18:03,546 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,546 DEBUG [Thread-2111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:18:03,598 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-11-26T14:18:03,598 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T14:18:03,598 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,598 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 176 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2433 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2307 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2322 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2451 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2306 2024-11-26T14:18:03,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T14:18:03,599 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:18:03,599 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:18:03,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:03,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:18:03,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T14:18:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:03,606 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630683606"}]},"ts":"1732630683606"} 2024-11-26T14:18:03,608 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T14:18:03,608 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T14:18:03,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T14:18:03,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, UNASSIGN}] 2024-11-26T14:18:03,611 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, UNASSIGN 2024-11-26T14:18:03,612 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=140 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=CLOSING, regionLocation=19945b63c2b4,40025,1732630490995 2024-11-26T14:18:03,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, UNASSIGN because future has completed 2024-11-26T14:18:03,613 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:18:03,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995}] 2024-11-26T14:18:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:03,766 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(122): Close a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:03,766 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:18:03,766 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1722): Closing a053eb1b7cd48562bf65d3c9969f8a5b, disabling compactions & flushes 2024-11-26T14:18:03,766 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. after waiting 0 ms 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:18:03,767 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(2902): Flushing a053eb1b7cd48562bf65d3c9969f8a5b 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=A 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=B 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a053eb1b7cd48562bf65d3c9969f8a5b, store=C 2024-11-26T14:18:03,767 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T14:18:03,774 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112697e715b759d84774b2608e8064662df6_a053eb1b7cd48562bf65d3c9969f8a5b is 50, key is test_row_0/A:col10/1732630683595/Put/seqid=0 2024-11-26T14:18:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742421_1597 (size=9914) 2024-11-26T14:18:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:04,177 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:18:04,180 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112697e715b759d84774b2608e8064662df6_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112697e715b759d84774b2608e8064662df6_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:04,181 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9eb1d2d3a2064467b9b6b146c07d9008, store: [table=TestAcidGuarantees family=A region=a053eb1b7cd48562bf65d3c9969f8a5b] 2024-11-26T14:18:04,182 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9eb1d2d3a2064467b9b6b146c07d9008 is 175, key is test_row_0/A:col10/1732630683595/Put/seqid=0 2024-11-26T14:18:04,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742422_1598 (size=22561) 2024-11-26T14:18:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:04,587 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=616, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9eb1d2d3a2064467b9b6b146c07d9008 2024-11-26T14:18:04,600 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b2b7e39362dd42839f056f118deda466 is 50, key is test_row_0/B:col10/1732630683595/Put/seqid=0 2024-11-26T14:18:04,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742423_1599 (size=9857) 2024-11-26T14:18:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:05,006 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b2b7e39362dd42839f056f118deda466 2024-11-26T14:18:05,011 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d04ddf828b1a464592f0d8bf23ca66f7 is 50, key is test_row_0/C:col10/1732630683595/Put/seqid=0 2024-11-26T14:18:05,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742424_1600 (size=9857) 2024-11-26T14:18:05,414 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d04ddf828b1a464592f0d8bf23ca66f7 2024-11-26T14:18:05,418 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/A/9eb1d2d3a2064467b9b6b146c07d9008 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9eb1d2d3a2064467b9b6b146c07d9008 2024-11-26T14:18:05,421 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9eb1d2d3a2064467b9b6b146c07d9008, entries=100, sequenceid=616, filesize=22.0 K 2024-11-26T14:18:05,422 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/B/b2b7e39362dd42839f056f118deda466 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b2b7e39362dd42839f056f118deda466 2024-11-26T14:18:05,425 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b2b7e39362dd42839f056f118deda466, entries=100, sequenceid=616, filesize=9.6 K 2024-11-26T14:18:05,425 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/.tmp/C/d04ddf828b1a464592f0d8bf23ca66f7 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d04ddf828b1a464592f0d8bf23ca66f7 2024-11-26T14:18:05,428 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d04ddf828b1a464592f0d8bf23ca66f7, entries=100, sequenceid=616, filesize=9.6 K 2024-11-26T14:18:05,429 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a053eb1b7cd48562bf65d3c9969f8a5b in 1662ms, sequenceid=616, compaction requested=false 2024-11-26T14:18:05,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59] to archive 2024-11-26T14:18:05,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:18:05,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/801b11e043964a0f8fe8add0ac962ff8 2024-11-26T14:18:05,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c600921b83b2411f99a0cd30218652bd 2024-11-26T14:18:05,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/32bf4fe5af074404a1fe7ed641ed01e0 2024-11-26T14:18:05,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/429b007d508e4742a593b9a4dcce2924 2024-11-26T14:18:05,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/10dc110ee9de48aa86ab88d6ca250826 2024-11-26T14:18:05,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a923521291074ad997514d7f06d4bd44 2024-11-26T14:18:05,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/30d7e070c0c0450b80125f0e5872234c 2024-11-26T14:18:05,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7d8801b5571d4be3bd19c9f1e934a8bd 2024-11-26T14:18:05,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/f975c9ce2cf34615b9dc063d08eb4404 2024-11-26T14:18:05,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0789cbaee5fa49ebaa8ae0ff4c958aa7 2024-11-26T14:18:05,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/1e39fe81f2634913a22137d318473aaf 2024-11-26T14:18:05,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c480fa8a91e84ac68f2b70985c9eb8e3 2024-11-26T14:18:05,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6b998a8f859a4f2684652a5892e21cb6 2024-11-26T14:18:05,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/3462fed406f0456c88dd06553f52bd4d 2024-11-26T14:18:05,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ca6799c6d67b4c259c0beac32ea7827f 2024-11-26T14:18:05,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a7fc324716d94685884ffdeb54f1fdc6 2024-11-26T14:18:05,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/c64695de939548f4b63ec10b9a2b827a 2024-11-26T14:18:05,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/0daa5aae7e76465aa6bf801b22bcf26f 2024-11-26T14:18:05,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/fac2493da37346908a01b00663f118a9 2024-11-26T14:18:05,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/74de777b79c24993b31adf0e495ea392 2024-11-26T14:18:05,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/16b5f4a5fab3442b801cfcd19c4d4eaf 2024-11-26T14:18:05,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9ad6870544e24b778cae1d6508a428ad 2024-11-26T14:18:05,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/ab12e6eece4c4c8eac6feb996d2e3480 2024-11-26T14:18:05,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/2121f920c3d548b2858de45d7e5398f0 2024-11-26T14:18:05,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/b82e49dfdc604562a3e932b21617fad5 2024-11-26T14:18:05,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/a12a47c4c2fc497f887119a7d4f9f22e 2024-11-26T14:18:05,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/bef66f8fac2a4e6b9143bca6f79cc984 2024-11-26T14:18:05,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9a09f232e78d48ae886a07acc4cfde39 2024-11-26T14:18:05,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/465ca332c9f1462a8089890b4ab742f1 2024-11-26T14:18:05,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/7585530e57a24cfbb90fac8106b892c3 2024-11-26T14:18:05,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/775e5f1ffc824aa690880e7eb1abe736 2024-11-26T14:18:05,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/6519efacd300490fae70852a327234cb 2024-11-26T14:18:05,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/395e8fc2c379450daa30cb9f2bf4ad59 2024-11-26T14:18:05,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92d597b2aec04cf7afab295270d1cdaf, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/99243259103148898439bfc25e6049c8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/612a332cb72345258db13f6552ea594c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ac769e1f94364d83a792586e57e08e57, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c57fa37290e3411ea978301aec7985b7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6e2ea561352e4fcb901e1539e0865731, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/bdbc17e0ae4c413686cb09e365939906, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/805e5aa397d14366847bad6d8aa788e9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/9ec23905ee1a4dbcaf31a28b59389899, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360] to archive 2024-11-26T14:18:05,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:18:05,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4628e28d078e465683009423dde244de 2024-11-26T14:18:05,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e3e843981f014cb6b3f18532d05f2c15 2024-11-26T14:18:05,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92d597b2aec04cf7afab295270d1cdaf to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92d597b2aec04cf7afab295270d1cdaf 2024-11-26T14:18:05,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/2f98dc4cb3204a748ece1ddb0090d4d3 2024-11-26T14:18:05,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/10fdf05cfa7e4bd3a4143cca437f8387 2024-11-26T14:18:05,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/92ca841131b74baa8633622527e94b29 2024-11-26T14:18:05,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/99243259103148898439bfc25e6049c8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/99243259103148898439bfc25e6049c8 2024-11-26T14:18:05,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/aa51b36e4b9a47578de6c1a071363779 2024-11-26T14:18:05,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c7a6ac2ea75845c39e2d5656aa035fa6 2024-11-26T14:18:05,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/612a332cb72345258db13f6552ea594c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/612a332cb72345258db13f6552ea594c 2024-11-26T14:18:05,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/05c896f0d2bd43ed8126e04315b89f53 2024-11-26T14:18:05,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/e1431153ffd94a4d9ad8a29a97bf5667 2024-11-26T14:18:05,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ac769e1f94364d83a792586e57e08e57 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ac769e1f94364d83a792586e57e08e57 2024-11-26T14:18:05,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/d18a0324889040bab0d17834e615b583 2024-11-26T14:18:05,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/13bc365575ec449c9e4ed05dfc856ace 2024-11-26T14:18:05,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cf8adf1938c04b72b75a193254fd9aaa 2024-11-26T14:18:05,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c57fa37290e3411ea978301aec7985b7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c57fa37290e3411ea978301aec7985b7 2024-11-26T14:18:05,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b9e82eb06f6d4c10ac473db37db10c5a 2024-11-26T14:18:05,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/0fbc5e9277cb419ca0b770b3629a8daa 2024-11-26T14:18:05,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6e2ea561352e4fcb901e1539e0865731 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6e2ea561352e4fcb901e1539e0865731 2024-11-26T14:18:05,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/df2de83b5ad14a99933a883c9cfd8a15 2024-11-26T14:18:05,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/82cc88ce393340d994471281a3cd06ce 2024-11-26T14:18:05,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/bdbc17e0ae4c413686cb09e365939906 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/bdbc17e0ae4c413686cb09e365939906 2024-11-26T14:18:05,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6bd05e2d385043278b4bf37f42249cd8 2024-11-26T14:18:05,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/c696c80a5d604760a702afffe2e8a154 2024-11-26T14:18:05,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/805e5aa397d14366847bad6d8aa788e9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/805e5aa397d14366847bad6d8aa788e9 2024-11-26T14:18:05,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/cc521e3b73044c65abe82fe9e1c726e1 2024-11-26T14:18:05,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/60d8696689004c609dcd042f4f9dc819 2024-11-26T14:18:05,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/9ec23905ee1a4dbcaf31a28b59389899 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/9ec23905ee1a4dbcaf31a28b59389899 2024-11-26T14:18:05,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/4c35d887011f4dce8e716a919722e687 2024-11-26T14:18:05,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/ef548808f208477a96cbe1176dee4e49 2024-11-26T14:18:05,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/12e36f732dd14c92b1d84c806a287670 2024-11-26T14:18:05,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/6897c2423873418c8c9850374575b360 2024-11-26T14:18:05,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/aa54ee5f449f4082b113a77bf1ecd505, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/65bd853cbc5247718471e05defbcf91a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/a0493fd696e74087b123100869cefbb6, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/25306b9761a4462baea5145adf4cc13f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/bd5a196fe8fb48c9975483a1e71327f9, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/594d6a738adb43f9b0050a0e1aa66396, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3c5c173bb10f45f9ad812f50bd0f9d8e, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4438b53dcf0b4d248eeab40547b84d8a, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588] to archive 2024-11-26T14:18:05,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:18:05,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c13fba2b74b543e482b2075cd2426974 2024-11-26T14:18:05,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/5dfcdef2f7264597897d8fd4be9a3632 2024-11-26T14:18:05,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/aa54ee5f449f4082b113a77bf1ecd505 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/aa54ee5f449f4082b113a77bf1ecd505 2024-11-26T14:18:05,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8727ea79770e471f8d3ad07b4b5f2648 2024-11-26T14:18:05,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/de66ca4d15e14ec99fa2592996bd78ca 2024-11-26T14:18:05,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/b9900cdf286d47189d05e31909521e73 2024-11-26T14:18:05,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/65bd853cbc5247718471e05defbcf91a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/65bd853cbc5247718471e05defbcf91a 2024-11-26T14:18:05,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f920a74b2fdc45d5bf2743e89c1c67a7 2024-11-26T14:18:05,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2efbdb1468a2481ba57f29ec9de3d22e 2024-11-26T14:18:05,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/a0493fd696e74087b123100869cefbb6 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/a0493fd696e74087b123100869cefbb6 2024-11-26T14:18:05,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/f5a8dfc1a6ca40b7960ac08ff092ccba 2024-11-26T14:18:05,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/77307cbe3d9f43118fc16e1b0ccf4d1c 2024-11-26T14:18:05,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/25306b9761a4462baea5145adf4cc13f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/25306b9761a4462baea5145adf4cc13f 2024-11-26T14:18:05,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4483652a2ea04000a75c223d612f1064 2024-11-26T14:18:05,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/0ff63a771a9d40a0a131fe0dea10b7ab 2024-11-26T14:18:05,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/531147ce3d734761abf71ba3da34c3a0 2024-11-26T14:18:05,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/bd5a196fe8fb48c9975483a1e71327f9 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/bd5a196fe8fb48c9975483a1e71327f9 2024-11-26T14:18:05,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3931fca3d62c4d07a718d24385c3b48b 2024-11-26T14:18:05,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d4173ad42be249cb94f6848570a0908f 2024-11-26T14:18:05,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/abae2f3b96624bab80ea2713bb3cea04 2024-11-26T14:18:05,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/594d6a738adb43f9b0050a0e1aa66396 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/594d6a738adb43f9b0050a0e1aa66396 2024-11-26T14:18:05,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3abf68076abc40268240d2fedcbd4290 2024-11-26T14:18:05,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/8eeaf08ff9b049028c1e927a152d6a91 2024-11-26T14:18:05,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3c5c173bb10f45f9ad812f50bd0f9d8e to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3c5c173bb10f45f9ad812f50bd0f9d8e 2024-11-26T14:18:05,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/6cc164e8069948ccbc36467f374b92b2 2024-11-26T14:18:05,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/3849510335e24455b3189995db52908b 2024-11-26T14:18:05,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/c57b77d9f2384ef998c4e69ec6d4e894 2024-11-26T14:18:05,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4438b53dcf0b4d248eeab40547b84d8a to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/4438b53dcf0b4d248eeab40547b84d8a 2024-11-26T14:18:05,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/2cb240349cda4b6099686dcc12a9a234 2024-11-26T14:18:05,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/1e0a55ce3c344866bf4d5be7d552ae2d 2024-11-26T14:18:05,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/107afb74dca44baf961751ed51a00b7c 2024-11-26T14:18:05,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/186406512dfe4a3db729374e5736c588 2024-11-26T14:18:05,515 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits/619.seqid, newMaxSeqId=619, maxSeqId=4 2024-11-26T14:18:05,516 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. 2024-11-26T14:18:05,516 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1676): Region close journal for a053eb1b7cd48562bf65d3c9969f8a5b: Waiting for close lock at 1732630683766Running coprocessor pre-close hooks at 1732630683766Disabling compacts and flushes for region at 1732630683766Disabling writes for close at 1732630683767 (+1 ms)Obtaining lock to block concurrent updates at 1732630683767Preparing flush snapshotting stores in a053eb1b7cd48562bf65d3c9969f8a5b at 1732630683767Finished memstore snapshotting TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732630683767Flushing stores of TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b. at 1732630683768 (+1 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/A: creating writer at 1732630683768Flushing mob file a053eb1b7cd48562bf65d3c9969f8a5b/A: appending metadata at 1732630683774 (+6 ms)Flushing mob file a053eb1b7cd48562bf65d3c9969f8a5b/A: closing flushed file at 1732630683774Flushing a053eb1b7cd48562bf65d3c9969f8a5b/A: appending metadata at 1732630684181 (+407 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/A: closing flushed file at 1732630684182 (+1 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/B: creating writer at 1732630684594 (+412 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/B: appending metadata at 1732630684599 (+5 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/B: closing flushed file at 1732630684599Flushing a053eb1b7cd48562bf65d3c9969f8a5b/C: creating writer at 1732630685008 (+409 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/C: appending metadata at 1732630685010 (+2 ms)Flushing a053eb1b7cd48562bf65d3c9969f8a5b/C: closing flushed file at 1732630685010Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13efcdfc: reopening flushed file at 1732630685418 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@121fdd52: reopening flushed file at 1732630685422 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@efa84ae: reopening flushed file at 1732630685425 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a053eb1b7cd48562bf65d3c9969f8a5b in 1662ms, sequenceid=616, compaction requested=false at 1732630685429 (+4 ms)Writing region close event to WAL at 1732630685513 (+84 ms)Running coprocessor post-close hooks at 1732630685516 (+3 ms)Closed at 1732630685516 2024-11-26T14:18:05,518 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(157): Closed a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=140 updating hbase:meta row=a053eb1b7cd48562bf65d3c9969f8a5b, regionState=CLOSED 2024-11-26T14:18:05,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 because future has completed 2024-11-26T14:18:05,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-26T14:18:05,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; CloseRegionProcedure a053eb1b7cd48562bf65d3c9969f8a5b, server=19945b63c2b4,40025,1732630490995 in 1.9070 sec 2024-11-26T14:18:05,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-26T14:18:05,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a053eb1b7cd48562bf65d3c9969f8a5b, UNASSIGN in 1.9110 sec 2024-11-26T14:18:05,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-26T14:18:05,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-26T14:18:05,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732630685525"}]},"ts":"1732630685525"} 2024-11-26T14:18:05,526 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T14:18:05,527 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T14:18:05,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-26T14:18:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-26T14:18:05,753 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:18:05,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T14:18:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,758 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=142, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T14:18:05,759 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=142, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,763 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,765 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C, FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits] 2024-11-26T14:18:05,768 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/62fdf575bae04ddda1237932a85b0396 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/62fdf575bae04ddda1237932a85b0396 2024-11-26T14:18:05,768 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9eb1d2d3a2064467b9b6b146c07d9008 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/A/9eb1d2d3a2064467b9b6b146c07d9008 2024-11-26T14:18:05,770 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/972941db3cc04831a4f626e836ef6b06 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/972941db3cc04831a4f626e836ef6b06 2024-11-26T14:18:05,771 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b2b7e39362dd42839f056f118deda466 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/B/b2b7e39362dd42839f056f118deda466 2024-11-26T14:18:05,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/9b8d63c111e14fa8887a2f4d875b24ed to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/9b8d63c111e14fa8887a2f4d875b24ed 2024-11-26T14:18:05,774 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d04ddf828b1a464592f0d8bf23ca66f7 to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/C/d04ddf828b1a464592f0d8bf23ca66f7 2024-11-26T14:18:05,775 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits/619.seqid to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b/recovered.edits/619.seqid 2024-11-26T14:18:05,776 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/default/TestAcidGuarantees/a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,776 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T14:18:05,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:18:05,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T14:18:05,778 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a3f2361889b418c86f21ad15d37fe54_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260a3f2361889b418c86f21ad15d37fe54_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,779 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126313688aa7f1a489788a682d8eb674b04_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126313688aa7f1a489788a682d8eb674b04_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,780 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d34075f66d7481abab36fc89b2dd537_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263d34075f66d7481abab36fc89b2dd537_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263e68d1f8fe20497bbcceee3dd866e688_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263e68d1f8fe20497bbcceee3dd866e688_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fa548066526422b9dfc57c8819cd5d1_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263fa548066526422b9dfc57c8819cd5d1_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,782 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112649de8aa30f434953bca4a9bd2abfccd4_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112649de8aa30f434953bca4a9bd2abfccd4_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265264bd07cbca47d89c34294b1b13cf0b_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265264bd07cbca47d89c34294b1b13cf0b_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c209e5e90ac436799533919097a79d3_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265c209e5e90ac436799533919097a79d3_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666dace0704424f608bdf0b85f0943249_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112666dace0704424f608bdf0b85f0943249_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,785 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266b2c5f0b6c384acd96de4c71fcb34282_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266b2c5f0b6c384acd96de4c71fcb34282_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,786 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d96438de62a4195bbef8a328161f3c1_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d96438de62a4195bbef8a328161f3c1_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,787 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269029205f11794ce7810d35acaf2fcfbd_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269029205f11794ce7810d35acaf2fcfbd_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,788 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112690cc8ebc4c744df8b459dd02ec0451df_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112690cc8ebc4c744df8b459dd02ec0451df_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,789 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112697e715b759d84774b2608e8064662df6_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112697e715b759d84774b2608e8064662df6_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,789 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c5ce2e58e784099a9d9101e7e2f792e_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269c5ce2e58e784099a9d9101e7e2f792e_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,790 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d3e94eb830e4e0fa449d70c56c592dc_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269d3e94eb830e4e0fa449d70c56c592dc_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,791 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b277c0a6ee7740ab8aca1c244f794351_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b277c0a6ee7740ab8aca1c244f794351_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,792 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126df723422e81f4e9191f874554fe224e9_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126df723422e81f4e9191f874554fe224e9_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,792 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e76c8a8915e744f7b1a6e1e2c2b0c240_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e76c8a8915e744f7b1a6e1e2c2b0c240_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,793 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eab04c8cafb54ed2ab217bf2b371e754_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eab04c8cafb54ed2ab217bf2b371e754_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,794 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f0a68064904d48c889ad1bd4cc732688_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f0a68064904d48c889ad1bd4cc732688_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,794 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1cd51b1cc564fa5a20503549ffa9fc9_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f1cd51b1cc564fa5a20503549ffa9fc9_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,795 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f76f0f5fd60b418d8773a3f305f10b96_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f76f0f5fd60b418d8773a3f305f10b96_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,795 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8d932e9aeb240008ff85d1bed9846a5_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8d932e9aeb240008ff85d1bed9846a5_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,796 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ffed7d9ce0344298bcc0a8849b768da6_a053eb1b7cd48562bf65d3c9969f8a5b to hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ffed7d9ce0344298bcc0a8849b768da6_a053eb1b7cd48562bf65d3c9969f8a5b 2024-11-26T14:18:05,796 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T14:18:05,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=142, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,800 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T14:18:05,801 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T14:18:05,802 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=142, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,802 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T14:18:05,802 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732630685802"}]},"ts":"9223372036854775807"} 2024-11-26T14:18:05,803 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T14:18:05,803 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a053eb1b7cd48562bf65d3c9969f8a5b, NAME => 'TestAcidGuarantees,,1732630651994.a053eb1b7cd48562bf65d3c9969f8a5b.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T14:18:05,803 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T14:18:05,803 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732630685803"}]},"ts":"9223372036854775807"} 2024-11-26T14:18:05,805 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T14:18:05,805 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=142, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T14:18:05,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-26T14:18:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37705 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T14:18:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T14:18:05,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T14:18:05,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=958 (was 1105), ProcessCount=11 (was 11), AvailableMemoryMB=5089 (was 6388) 2024-11-26T14:18:05,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T14:18:05,871 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:18:05,871 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:18:05,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:05,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:05,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:18:05,871 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T14:18:05,871 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-26T14:18:05,872 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=19945b63c2b4,37705,1732630489590 2024-11-26T14:18:05,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T14:18:05,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T14:18:05,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:18:05,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:18:05,914 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T14:18:05,915 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:18:05,915 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:18:05,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:05,915 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:18:05,915 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:18:05,915 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '19945b63c2b4,40025,1732630490995' ***** 2024-11-26T14:18:05,915 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T14:18:05,916 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T14:18:05,916 INFO [RS:0;19945b63c2b4:40025 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T14:18:05,916 INFO [RS:0;19945b63c2b4:40025 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T14:18:05,916 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(959): stopping server 19945b63c2b4,40025,1732630490995 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;19945b63c2b4:40025. 2024-11-26T14:18:05,917 DEBUG [RS:0;19945b63c2b4:40025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:18:05,917 DEBUG [RS:0;19945b63c2b4:40025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T14:18:05,917 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T14:18:05,918 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T14:18:05,918 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T14:18:05,919 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:05,919 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T14:18:05,919 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T14:18:05,919 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T14:18:05,919 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T14:18:05,919 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T14:18:05,919 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T14:18:05,942 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/info/49557e0cfefd48159adc92319c56a86a is 95, key is TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2./info:/1732630523209/DeleteFamily/seqid=0 2024-11-26T14:18:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742425_1601 (size=6514) 2024-11-26T14:18:05,946 INFO [regionserver/19945b63c2b4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:18:06,119 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:06,319 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:06,346 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/info/49557e0cfefd48159adc92319c56a86a 2024-11-26T14:18:06,364 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/ns/a6fe9ced106c485391bf0322492e5871 is 93, key is TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2./ns:/1732630523155/DeleteFamily/seqid=0 2024-11-26T14:18:06,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742426_1602 (size=5906) 2024-11-26T14:18:06,397 INFO [regionserver/19945b63c2b4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T14:18:06,397 INFO [regionserver/19945b63c2b4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T14:18:06,519 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:06,719 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:06,768 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/ns/a6fe9ced106c485391bf0322492e5871 2024-11-26T14:18:06,791 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/rep_barrier/a3596ac29e364de8ae93f5e7368676b8 is 102, key is TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2./rep_barrier:/1732630523155/DeleteFamily/seqid=0 2024-11-26T14:18:06,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742427_1603 (size=6025) 2024-11-26T14:18:06,920 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:07,120 DEBUG [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T14:18:07,194 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/rep_barrier/a3596ac29e364de8ae93f5e7368676b8 2024-11-26T14:18:07,210 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/table/81fbae0521254b3093a748d6a0bfa269 is 96, key is TestAcidGuarantees,,1732630496238.c77bbfdf456d364621817f40b0d9aed2./table:/1732630523155/DeleteFamily/seqid=0 2024-11-26T14:18:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742428_1604 (size=5936) 2024-11-26T14:18:07,215 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/table/81fbae0521254b3093a748d6a0bfa269 2024-11-26T14:18:07,222 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/info/49557e0cfefd48159adc92319c56a86a as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/info/49557e0cfefd48159adc92319c56a86a 2024-11-26T14:18:07,225 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/info/49557e0cfefd48159adc92319c56a86a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T14:18:07,226 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/ns/a6fe9ced106c485391bf0322492e5871 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/ns/a6fe9ced106c485391bf0322492e5871 2024-11-26T14:18:07,228 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/ns/a6fe9ced106c485391bf0322492e5871, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T14:18:07,229 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/rep_barrier/a3596ac29e364de8ae93f5e7368676b8 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/rep_barrier/a3596ac29e364de8ae93f5e7368676b8 2024-11-26T14:18:07,232 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/rep_barrier/a3596ac29e364de8ae93f5e7368676b8, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T14:18:07,233 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/.tmp/table/81fbae0521254b3093a748d6a0bfa269 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/table/81fbae0521254b3093a748d6a0bfa269 2024-11-26T14:18:07,237 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/table/81fbae0521254b3093a748d6a0bfa269, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T14:18:07,238 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1318ms, sequenceid=90, compaction requested=false 2024-11-26T14:18:07,242 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T14:18:07,243 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T14:18:07,243 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T14:18:07,243 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732630685918Running coprocessor pre-close hooks at 1732630685919 (+1 ms)Disabling compacts and flushes for region at 1732630685919Disabling writes for close at 1732630685919Obtaining lock to block concurrent updates at 1732630685919Preparing flush snapshotting stores in 1588230740 at 1732630685919Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732630685920 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732630685920Flushing 1588230740/info: creating writer at 1732630685921 (+1 ms)Flushing 1588230740/info: appending metadata at 1732630685941 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732630685941Flushing 1588230740/ns: creating writer at 1732630686349 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732630686364 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732630686364Flushing 1588230740/rep_barrier: creating writer at 1732630686773 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732630686791 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732630686791Flushing 1588230740/table: creating writer at 1732630687197 (+406 ms)Flushing 1588230740/table: appending metadata at 1732630687210 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732630687210Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3da34a95: reopening flushed file at 1732630687221 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dac6f4: reopening flushed file at 1732630687225 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4747ba3d: reopening flushed file at 1732630687228 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d8595e5: reopening flushed file at 1732630687232 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1318ms, sequenceid=90, compaction requested=false at 1732630687238 (+6 ms)Writing region close event to WAL at 1732630687239 (+1 ms)Running coprocessor post-close hooks at 1732630687242 (+3 ms)Closed at 1732630687243 (+1 ms) 2024-11-26T14:18:07,243 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T14:18:07,321 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(976): stopping server 19945b63c2b4,40025,1732630490995; all regions closed. 2024-11-26T14:18:07,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741834_1010 (size=25344) 2024-11-26T14:18:07,327 DEBUG [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/oldWALs 2024-11-26T14:18:07,327 INFO [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 19945b63c2b4%2C40025%2C1732630490995.meta:.meta(num 1732630495491) 2024-11-26T14:18:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741833_1009 (size=19732640) 2024-11-26T14:18:07,331 DEBUG [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/oldWALs 2024-11-26T14:18:07,331 INFO [RS:0;19945b63c2b4:40025 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 19945b63c2b4%2C40025%2C1732630490995:(num 1732630494636) 2024-11-26T14:18:07,331 DEBUG [RS:0;19945b63c2b4:40025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:18:07,331 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:18:07,331 INFO [RS:0;19945b63c2b4:40025 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:18:07,331 INFO [RS:0;19945b63c2b4:40025 {}] hbase.ChoreService(370): Chore service for: regionserver/19945b63c2b4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T14:18:07,331 INFO [RS:0;19945b63c2b4:40025 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:18:07,331 INFO [regionserver/19945b63c2b4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:18:07,332 INFO [RS:0;19945b63c2b4:40025 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40025 2024-11-26T14:18:07,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/19945b63c2b4,40025,1732630490995 2024-11-26T14:18:07,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:18:07,345 INFO [RS:0;19945b63c2b4:40025 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:18:07,346 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [19945b63c2b4,40025,1732630490995] 2024-11-26T14:18:07,355 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/19945b63c2b4,40025,1732630490995 already deleted, retry=false 2024-11-26T14:18:07,356 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 19945b63c2b4,40025,1732630490995 expired; onlineServers=0 2024-11-26T14:18:07,356 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '19945b63c2b4,37705,1732630489590' ***** 2024-11-26T14:18:07,356 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T14:18:07,356 INFO [M:0;19945b63c2b4:37705 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:18:07,356 INFO [M:0;19945b63c2b4:37705 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:18:07,356 DEBUG [M:0;19945b63c2b4:37705 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T14:18:07,356 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T14:18:07,356 DEBUG [M:0;19945b63c2b4:37705 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T14:18:07,356 DEBUG [master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732630494069 {}] cleaner.HFileCleaner(306): Exit Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732630494069,5,FailOnTimeoutGroup] 2024-11-26T14:18:07,356 DEBUG [master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732630494064 {}] cleaner.HFileCleaner(306): Exit Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732630494064,5,FailOnTimeoutGroup] 2024-11-26T14:18:07,356 INFO [M:0;19945b63c2b4:37705 {}] hbase.ChoreService(370): Chore service for: master/19945b63c2b4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T14:18:07,356 INFO [M:0;19945b63c2b4:37705 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:18:07,357 DEBUG [M:0;19945b63c2b4:37705 {}] master.HMaster(1795): Stopping service threads 2024-11-26T14:18:07,357 INFO [M:0;19945b63c2b4:37705 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T14:18:07,357 INFO [M:0;19945b63c2b4:37705 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T14:18:07,357 ERROR [M:0;19945b63c2b4:37705 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:46747 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46747,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T14:18:07,357 INFO [M:0;19945b63c2b4:37705 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T14:18:07,357 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T14:18:07,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T14:18:07,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:18:07,366 DEBUG [M:0;19945b63c2b4:37705 {}] zookeeper.ZKUtil(347): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T14:18:07,366 WARN [M:0;19945b63c2b4:37705 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T14:18:07,367 INFO [M:0;19945b63c2b4:37705 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/.lastflushedseqids 2024-11-26T14:18:07,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742429_1605 (size=72) 2024-11-26T14:18:07,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:18:07,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40025-0x101777f21520001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:18:07,457 INFO [RS:0;19945b63c2b4:40025 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:18:07,457 INFO [RS:0;19945b63c2b4:40025 {}] regionserver.HRegionServer(1031): Exiting; stopping=19945b63c2b4,40025,1732630490995; zookeeper connection closed. 2024-11-26T14:18:07,458 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f67e358 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f67e358 2024-11-26T14:18:07,459 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T14:18:07,776 INFO [M:0;19945b63c2b4:37705 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T14:18:07,776 INFO [M:0;19945b63c2b4:37705 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T14:18:07,776 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T14:18:07,776 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:18:07,776 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:18:07,776 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T14:18:07,776 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:18:07,776 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=703.88 KB heapSize=857.46 KB 2024-11-26T14:18:07,795 DEBUG [M:0;19945b63c2b4:37705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c72f18b4afd94adea88d7970b05b3031 is 82, key is hbase:meta,,1/info:regioninfo/1732630495668/Put/seqid=0 2024-11-26T14:18:07,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742430_1606 (size=5672) 2024-11-26T14:18:08,198 INFO [M:0;19945b63c2b4:37705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1897 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c72f18b4afd94adea88d7970b05b3031 2024-11-26T14:18:08,217 DEBUG [M:0;19945b63c2b4:37705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/971a30b4f71a43dbbe949a8e74fe7956 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x1B/proc:d/1732630525457/Put/seqid=0 2024-11-26T14:18:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742431_1607 (size=40471) 2024-11-26T14:18:08,621 INFO [M:0;19945b63c2b4:37705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=703.32 KB at sequenceid=1897 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/971a30b4f71a43dbbe949a8e74fe7956 2024-11-26T14:18:08,625 INFO [M:0;19945b63c2b4:37705 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 971a30b4f71a43dbbe949a8e74fe7956 2024-11-26T14:18:08,639 DEBUG [M:0;19945b63c2b4:37705 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bb7140fc4424a78ba84722362380550 is 69, key is 19945b63c2b4,40025,1732630490995/rs:state/1732630494233/Put/seqid=0 2024-11-26T14:18:08,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073742432_1608 (size=5156) 2024-11-26T14:18:09,043 INFO [M:0;19945b63c2b4:37705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1897 (bloomFilter=true), to=hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bb7140fc4424a78ba84722362380550 2024-11-26T14:18:09,047 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c72f18b4afd94adea88d7970b05b3031 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c72f18b4afd94adea88d7970b05b3031 2024-11-26T14:18:09,050 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c72f18b4afd94adea88d7970b05b3031, entries=8, sequenceid=1897, filesize=5.5 K 2024-11-26T14:18:09,051 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/971a30b4f71a43dbbe949a8e74fe7956 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/971a30b4f71a43dbbe949a8e74fe7956 2024-11-26T14:18:09,055 INFO [M:0;19945b63c2b4:37705 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 971a30b4f71a43dbbe949a8e74fe7956 2024-11-26T14:18:09,055 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/971a30b4f71a43dbbe949a8e74fe7956, entries=142, sequenceid=1897, filesize=39.5 K 2024-11-26T14:18:09,056 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bb7140fc4424a78ba84722362380550 as hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bb7140fc4424a78ba84722362380550 2024-11-26T14:18:09,059 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46747/user/jenkins/test-data/f6539855-3826-e126-cf03-acf5005b8bcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bb7140fc4424a78ba84722362380550, entries=1, sequenceid=1897, filesize=5.0 K 2024-11-26T14:18:09,059 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(3140): Finished flush of dataSize ~703.88 KB/720768, heapSize ~857.16 KB/877736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=1897, compaction requested=false 2024-11-26T14:18:09,063 INFO [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:18:09,063 DEBUG [M:0;19945b63c2b4:37705 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732630687776Disabling compacts and flushes for region at 1732630687776Disabling writes for close at 1732630687776Obtaining lock to block concurrent updates at 1732630687776Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732630687776Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=720768, getHeapSize=877976, getOffHeapSize=0, getCellsCount=2064 at 1732630687777 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732630687777Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732630687777Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732630687794 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732630687794Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732630688202 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732630688217 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732630688217Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732630688625 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732630688639 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732630688639Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@663bbc3: reopening flushed file at 1732630689046 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d74902a: reopening flushed file at 1732630689050 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a38a3e8: reopening flushed file at 1732630689055 (+5 ms)Finished flush of dataSize ~703.88 KB/720768, heapSize ~857.16 KB/877736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=1897, compaction requested=false at 1732630689059 (+4 ms)Writing region close event to WAL at 1732630689063 (+4 ms)Closed at 1732630689063 2024-11-26T14:18:09,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46501 is added to blk_1073741830_1006 (size=845160) 2024-11-26T14:18:09,067 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:18:09,067 INFO [M:0;19945b63c2b4:37705 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T14:18:09,067 INFO [M:0;19945b63c2b4:37705 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37705 2024-11-26T14:18:09,067 INFO [M:0;19945b63c2b4:37705 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:18:09,182 INFO [M:0;19945b63c2b4:37705 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:18:09,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:18:09,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37705-0x101777f21520000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:18:09,189 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:18:09,192 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:18:09,193 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:18:09,193 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:18:09,193 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.log.dir/,STOPPED} 2024-11-26T14:18:09,197 WARN [BP-17547768-172.17.0.3-1732630485263 heartbeating to localhost/127.0.0.1:46747 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T14:18:09,197 WARN [BP-17547768-172.17.0.3-1732630485263 heartbeating to localhost/127.0.0.1:46747 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-17547768-172.17.0.3-1732630485263 (Datanode Uuid 16b1d143-87c6-4d34-abc0-c7270ac6fb80) service to localhost/127.0.0.1:46747 2024-11-26T14:18:09,200 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T14:18:09,200 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T14:18:09,200 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data1/current/BP-17547768-172.17.0.3-1732630485263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:18:09,201 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/cluster_ca462def-78b4-bae6-3ea2-d52be145f79b/data/data2/current/BP-17547768-172.17.0.3-1732630485263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:18:09,201 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T14:18:09,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T14:18:09,211 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:18:09,211 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:18:09,212 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:18:09,212 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e73f4889-f888-c663-6bee-3d36c1400fca/hadoop.log.dir/,STOPPED} 2024-11-26T14:18:09,232 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T14:18:09,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down